From 6960e6da9cec3f6638121527c728305827ec12ab Mon Sep 17 00:00:00 2001 From: Gerd Hoffmann Date: Wed, 5 Sep 2018 08:04:43 +0200 Subject: drm: fix drm_mode_addfb() on big endian machines. Userspace on big endian machhines typically expects the ADDFB ioctl returns a big endian framebuffer. drm_mode_addfb() will call drm_mode_addfb2() unconditionally with little endian DRM_FORMAT_* values though, which is wrong. This patch fixes that. Drivers (both kernel and xorg) have quirks in place to deal with the broken drm_mode_addfb() behavior. Because of this we can't just change drm_mode_addfb() behavior for everybody without breaking things. Add the quirk_addfb_prefer_host_byte_order field to mode_config, so drivers can opt-in. Signed-off-by: Gerd Hoffmann Reviewed-by: Daniel Vetter Link: http://patchwork.freedesktop.org/patch/msgid/20180905060445.15008-5-kraxel@redhat.com --- drivers/gpu/drm/drm_framebuffer.c | 11 +++++++++++ include/drm/drm_mode_config.h | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/drivers/gpu/drm/drm_framebuffer.c b/drivers/gpu/drm/drm_framebuffer.c index 888c4d53cf23..f863f8a20f8c 100644 --- a/drivers/gpu/drm/drm_framebuffer.c +++ b/drivers/gpu/drm/drm_framebuffer.c @@ -124,6 +124,17 @@ int drm_mode_addfb(struct drm_device *dev, struct drm_mode_fb_cmd *or, r.pixel_format == DRM_FORMAT_XRGB2101010) r.pixel_format = DRM_FORMAT_XBGR2101010; + if (dev->mode_config.quirk_addfb_prefer_host_byte_order) { + if (r.pixel_format == DRM_FORMAT_XRGB8888) + r.pixel_format = DRM_FORMAT_HOST_XRGB8888; + if (r.pixel_format == DRM_FORMAT_ARGB8888) + r.pixel_format = DRM_FORMAT_HOST_ARGB8888; + if (r.pixel_format == DRM_FORMAT_RGB565) + r.pixel_format = DRM_FORMAT_HOST_RGB565; + if (r.pixel_format == DRM_FORMAT_XRGB1555) + r.pixel_format = DRM_FORMAT_HOST_XRGB1555; + } + ret = drm_mode_addfb2(dev, &r, file_priv); if (ret) return ret; diff --git a/include/drm/drm_mode_config.h b/include/drm/drm_mode_config.h index 5d29f4ba6f69..928e4172a0bb 100644 --- a/include/drm/drm_mode_config.h +++ b/include/drm/drm_mode_config.h @@ -811,6 +811,20 @@ struct drm_mode_config { uint32_t preferred_depth, prefer_shadow; bool quirk_addfb_prefer_xbgr_30bpp; + /** + * @quirk_addfb_prefer_host_byte_order: + * + * When set to true drm_mode_addfb() will pick host byte order + * pixel_format when calling drm_mode_addfb2(). This is how + * drm_mode_addfb() should have worked from day one. It + * didn't though, so we ended up with quirks in both kernel + * and userspace drivers to deal with the broken behavior. + * Simply fixing drm_mode_addfb() unconditionally would break + * these drivers, so add a quirk bit here to allow drivers + * opt-in. + */ + bool quirk_addfb_prefer_host_byte_order; + /** * @async_page_flip: Does this device support async flips on the primary * plane? -- cgit v1.2.3