From c0e13cbf5a56e1fdd1e4ce58ebdefb6d2904e4b3 Mon Sep 17 00:00:00 2001
From: Simon Ser <contact@emersion.fr>
Date: Tue, 2 Jun 2020 15:28:38 +0200
Subject: [PATCH] xwayland: only use linux-dmabuf if format/modifier was
 advertised
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously, linux-dmabuf was used unconditionally if the buffer had a
modifier. However creating a linux-dmabuf buffer with a format/modifier
which hasn't been advertised will fail.

Change xwl_glamor_gbm_get_wl_buffer_for_pixmap to use linux-dmabuf when
the format/modifier has been advertised only.

Signed-off-by: Simon Ser <contact@emersion.fr>
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1035
Tested-by: Emmanuel Gil Peyrot <linkmauve@linkmauve.fr>
Reviewed-by: Michel Dänzer <mdaenzer@redhat.com>
---
 hw/xwayland/xwayland-glamor-gbm.c | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/hw/xwayland/xwayland-glamor-gbm.c b/hw/xwayland/xwayland-glamor-gbm.c
index 379fe9c8c..7336c97a7 100644
--- a/hw/xwayland/xwayland-glamor-gbm.c
+++ b/hw/xwayland/xwayland-glamor-gbm.c
@@ -293,6 +293,9 @@ xwl_glamor_gbm_get_wl_buffer_for_pixmap(PixmapPtr pixmap,
     struct xwl_gbm_private *xwl_gbm = xwl_gbm_get(xwl_screen);
     unsigned short width = pixmap->drawable.width;
     unsigned short height = pixmap->drawable.height;
+    uint32_t format;
+    struct xwl_format *xwl_format = NULL;
+    Bool modifier_supported = FALSE;
     int prime_fd;
     int num_planes;
     uint32_t strides[4];
@@ -317,6 +320,8 @@ xwl_glamor_gbm_get_wl_buffer_for_pixmap(PixmapPtr pixmap,
     if (!xwl_pixmap->bo)
        return NULL;
 
+    format = wl_drm_format_for_depth(pixmap->drawable.depth);
+
     prime_fd = gbm_bo_get_fd(xwl_pixmap->bo);
     if (prime_fd == -1)
         return NULL;
@@ -335,7 +340,23 @@ xwl_glamor_gbm_get_wl_buffer_for_pixmap(PixmapPtr pixmap,
     offsets[0] = 0;
 #endif
 
-    if (xwl_gbm->dmabuf && modifier != DRM_FORMAT_MOD_INVALID) {
+    for (i = 0; i < xwl_screen->num_formats; i++) {
+       if (xwl_screen->formats[i].format == format) {
+          xwl_format = &xwl_screen->formats[i];
+          break;
+       }
+    }
+
+    if (xwl_format) {
+        for (i = 0; i < xwl_format->num_modifiers; i++) {
+            if (xwl_format->modifiers[i] == modifier) {
+                modifier_supported = TRUE;
+                break;
+            }
+        }
+    }
+
+    if (xwl_gbm->dmabuf && modifier_supported) {
         struct zwp_linux_buffer_params_v1 *params;
 
         params = zwp_linux_dmabuf_v1_create_params(xwl_gbm->dmabuf);
@@ -347,13 +368,12 @@ xwl_glamor_gbm_get_wl_buffer_for_pixmap(PixmapPtr pixmap,
 
         xwl_pixmap->buffer =
            zwp_linux_buffer_params_v1_create_immed(params, width, height,
-                                                   wl_drm_format_for_depth(pixmap->drawable.depth),
-                                                   0);
+                                                   format, 0);
         zwp_linux_buffer_params_v1_destroy(params);
     } else if (num_planes == 1) {
         xwl_pixmap->buffer =
             wl_drm_create_prime_buffer(xwl_gbm->drm, prime_fd, width, height,
-                                       wl_drm_format_for_depth(pixmap->drawable.depth),
+                                       format,
                                        0, gbm_bo_get_stride(xwl_pixmap->bo),
                                        0, 0,
                                        0, 0);