glamor: Add and use glamor_drawable_effective_depth helper

Consider the following window hierarchy, from ancestors to descendants:

 A
 |
 B
 |
 C

If both A & C have depth 32, but B has depth 24, C must effectively
behave as if it had depth 24, even if its backing pixmap has depth 32
as well.

Fixes the xmag issue described in the GitLab issue below.

Issue: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1564

glamor/glamor_copy.c

@@ -77,7 +77,7 @@ use_copyplane(DrawablePtr drawable, GCPtr gc, glamor_program *prog, void *arg)
     glamor_set_color(drawable, gc->bgPixel, prog->bg_uniform);
 
     /* XXX handle 2 10 10 10 and 1555 formats; presumably the pixmap private knows this? */
-    switch (args->src_drawable->depth) {
+    switch (glamor_drawable_effective_depth(args->src_drawable)) {
     case 30:
         glUniform4ui(prog->bitplane_uniform,
                      (args->bitplane >> 20) & 0x3ff,
@@ -235,7 +235,7 @@ glamor_copy_cpu_fbo(DrawablePtr src,
 
         PixmapPtr tmp_pix = fbCreatePixmap(screen, dst_pixmap->drawable.width,
                                            dst_pixmap->drawable.height,
-                                           dst->depth, 0);
+                                           glamor_drawable_effective_depth(dst), 0);
 
         if (!tmp_pix) {
             glamor_finish_access(src);
@@ -547,7 +547,7 @@ glamor_copy_fbo_fbo_temp(DrawablePtr src,
     tmp_pixmap = glamor_create_pixmap(screen,
                                       bounds.x2 - bounds.x1,
                                       bounds.y2 - bounds.y1,
-                                      src->depth, 0);
+                                      glamor_drawable_effective_depth(src), 0);
     if (!tmp_pixmap)
         goto bail;
 
@@ -757,7 +757,7 @@ glamor_copy_plane(DrawablePtr src, DrawablePtr dst, GCPtr gc,
                   int srcx, int srcy, int width, int height, int dstx, int dsty,
                   unsigned long bitplane)
 {
-    if ((bitplane & FbFullMask(src->depth)) == 0)
+    if ((bitplane & FbFullMask(glamor_drawable_effective_depth(src))) == 0)
         return miHandleExposures(src, dst, gc,
                                  srcx, srcy, width, height, dstx, dsty);
     return miDoCopy(src, dst, gc,

glamor/glamor_image.c

@@ -129,7 +129,7 @@ glamor_get_image_gl(DrawablePtr drawable, int x, int y, int w, int h,
                           -x, -y,
                           (uint8_t *) d, byte_stride);
 
-    if (!glamor_pm_is_solid(drawable->depth, plane_mask)) {
+    if (!glamor_pm_is_solid(glamor_drawable_effective_depth(drawable), plane_mask)) {
         FbStip pm = fbReplicatePixel(plane_mask, drawable->bitsPerPixel);
         FbStip *dst = (void *)d;
         uint32_t dstStride = byte_stride / sizeof(FbStip);

glamor/glamor_prepare.c

@@ -163,7 +163,7 @@ glamor_finish_access(DrawablePtr drawable)
         return;
 
     if (priv->pbo &&
-        !(drawable->depth == 24 && pixmap->drawable.depth == 32)) {
+        !(glamor_drawable_effective_depth(drawable) == 24 && pixmap->drawable.depth == 32)) {
         glBindBuffer(GL_PIXEL_UNPACK_BUFFER, priv->pbo);
         glUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
         pixmap->devPrivate.ptr = NULL;
@@ -179,7 +179,7 @@ glamor_finish_access(DrawablePtr drawable)
     RegionUninit(&priv->prepare_region);
 
     if (priv->pbo) {
-        if (drawable->depth == 24 && pixmap->drawable.depth == 32)
+        if (glamor_drawable_effective_depth(drawable) == 24 && pixmap->drawable.depth == 32)
             pixmap->devPrivate.ptr = NULL;
         else
             glBindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);

glamor/glamor_priv.h

@@ -513,6 +513,30 @@ glamor_pixmap_hcnt(glamor_pixmap_private *priv)
     for (box_index = 0; box_index < glamor_pixmap_hcnt(priv) *         \
              glamor_pixmap_wcnt(priv); box_index++)                    \
 
+static inline int
+glamor_drawable_effective_depth(DrawablePtr drawable)
+{
+    WindowPtr window;
+
+    if (drawable->type != DRAWABLE_WINDOW ||
+        drawable->depth != 32)
+        return drawable->depth;
+
+    window = (WindowPtr)drawable;
+    window = window->parent;
+    while (window && window->parent) {
+        /* A depth 32 window with any depth 24 ancestors (other than the root
+         * window) effectively behaves like depth 24
+         */
+        if (window->drawable.depth == 24)
+            return 24;
+
+        window = window->parent;
+    }
+
+    return 32;
+}
+
 /* GC private structure. Currently holds only any computed dash pixmap */
 
 typedef struct {

glamor/glamor_render.c

@@ -806,7 +806,7 @@ glamor_render_format_is_supported(PicturePtr picture)
         return TRUE;
 
     glamor_priv = glamor_get_screen_private(picture->pDrawable->pScreen);
-    f = &glamor_priv->formats[picture->pDrawable->depth];
+    f = &glamor_priv->formats[glamor_drawable_effective_depth(picture->pDrawable)];
 
     if (!f->rendering_supported)
         return FALSE;

glamor/glamor_transfer.c

@@ -41,7 +41,7 @@ glamor_upload_boxes(DrawablePtr drawable, BoxPtr in_boxes, int in_nbox,
     const struct glamor_format *f = glamor_format_for_pixmap(pixmap);
     char *tmp_bits = NULL;
 
-    if (drawable->depth == 24 && pixmap->drawable.depth == 32)
+    if (glamor_drawable_effective_depth(drawable) == 24 && pixmap->drawable.depth == 32)
         tmp_bits = xnfalloc(byte_stride * pixmap->drawable.height);
 
     glamor_make_current(glamor_priv);

glamor/glamor_transform.h

@@ -44,7 +44,8 @@ glamor_set_color(DrawablePtr    drawable,
                  GLint          uniform)
 {
     glamor_set_color_depth(drawable->pScreen,
-                           drawable->depth, pixel, uniform);
+                           glamor_drawable_effective_depth(drawable),
+                           pixel, uniform);
 }
 
 Bool