[PATCH] drm/virtio: Use XRGB8888 also for big endian systems

[Jocelyn Falempe]

Mesa doesn't support BGRX8888, that means most wayland compositors
doesn't work on big endian guests.

Also the colors are inverted when testing a s390x VM on a s390x host.
So I fixed the DRM_FORMAT -> VIRTIO_GPU_FORMAT on big endian guests.
It may break big-endian guest on little-endian host, but then the
fix should be in qemu, because we don't know the host endianess in
the guest VM.

Signed-off-by: Jocelyn Falempe <jfale...@redhat.com>
---
 drivers/gpu/drm/virtio/virtgpu_display.c |  4 ++--
 drivers/gpu/drm/virtio/virtgpu_gem.c     |  2 +-
 drivers/gpu/drm/virtio/virtgpu_plane.c   | 22 ++++++++++++----------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c 
b/drivers/gpu/drm/virtio/virtgpu_display.c
index 64baf2f22d9f0..3572a53ea2061 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -299,8 +299,8 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev,
        struct virtio_gpu_framebuffer *virtio_gpu_fb;
        int ret;
 
-       if (mode_cmd->pixel_format != DRM_FORMAT_HOST_XRGB8888 &&
-           mode_cmd->pixel_format != DRM_FORMAT_HOST_ARGB8888)
+       if (mode_cmd->pixel_format != DRM_FORMAT_XRGB8888 &&
+           mode_cmd->pixel_format != DRM_FORMAT_ARGB8888)
                return ERR_PTR(-ENOENT);
 
        /* lookup object associated with res handle */
diff --git a/drivers/gpu/drm/virtio/virtgpu_gem.c 
b/drivers/gpu/drm/virtio/virtgpu_gem.c
index 7db48d17ee3a8..601e06962530f 100644
--- a/drivers/gpu/drm/virtio/virtgpu_gem.c
+++ b/drivers/gpu/drm/virtio/virtgpu_gem.c
@@ -75,7 +75,7 @@ int virtio_gpu_mode_dumb_create(struct drm_file *file_priv,
        args->size = pitch * args->height;
        args->size = ALIGN(args->size, PAGE_SIZE);
 
-       params.format = virtio_gpu_translate_format(DRM_FORMAT_HOST_XRGB8888);
+       params.format = virtio_gpu_translate_format(DRM_FORMAT_XRGB8888);
        params.width = args->width;
        params.height = args->height;
        params.size = args->size;
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c 
b/drivers/gpu/drm/virtio/virtgpu_plane.c
index a72a2dbda031c..0ec6ecc96eb13 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -30,29 +30,31 @@
 #include "virtgpu_drv.h"
 
 static const uint32_t virtio_gpu_formats[] = {
-       DRM_FORMAT_HOST_XRGB8888,
+       DRM_FORMAT_XRGB8888,
 };
 
 static const uint32_t virtio_gpu_cursor_formats[] = {
-       DRM_FORMAT_HOST_ARGB8888,
+       DRM_FORMAT_ARGB8888,
 };
 
+#ifdef __BIG_ENDIAN
+#define VIRTIO_GPU_HOST_XRGB8888 VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM
+#define VIRTIO_GPU_HOST_ARGB8888 VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM
+#else
+#define VIRTIO_GPU_HOST_XRGB8888 VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM
+#define VIRTIO_GPU_HOST_ARGB8888 VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM
+#endif
+
 uint32_t virtio_gpu_translate_format(uint32_t drm_fourcc)
 {
        uint32_t format;
 
        switch (drm_fourcc) {
        case DRM_FORMAT_XRGB8888:
-               format = VIRTIO_GPU_FORMAT_B8G8R8X8_UNORM;
+               format = VIRTIO_GPU_HOST_XRGB8888;
                break;
        case DRM_FORMAT_ARGB8888:
-               format = VIRTIO_GPU_FORMAT_B8G8R8A8_UNORM;
-               break;
-       case DRM_FORMAT_BGRX8888:
-               format = VIRTIO_GPU_FORMAT_X8R8G8B8_UNORM;
-               break;
-       case DRM_FORMAT_BGRA8888:
-               format = VIRTIO_GPU_FORMAT_A8R8G8B8_UNORM;
+               format = VIRTIO_GPU_HOST_ARGB8888;
                break;
        default:
                /*

base-commit: 8befe8fa5a4e4b30787b17e078d9d7b5cb92ea19
-- 
2.46.0