From d1bbf82d72566e16e90800c77cf70fe5fc4ef2e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michel=20D=C3=A4nzer?= <mdaenzer@redhat.com>
Date: Fri, 5 Jan 2024 18:31:11 +0100
Subject: [PATCH] glamor: Don't override source alpha to 1.0 if it's used for
 blending

It caused an incorrect result of the blend operation.

Use glColorMask to prevent non-1.0 alpha channel values in a depth 32
pixmap backing an effective depth 24 window. For blending operations,
the expectation is that the destination drawable contains valid pixel
values, so the alpha channel should already be 1.0.

Fixes: d1f142891ef3 ("glamor: Ignore destination alpha as necessary for composite operation")
Issue: https://gitlab.gnome.org/GNOME/mutter/-/issues/3104
---
 glamor/glamor_render.c | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/glamor/glamor_render.c b/glamor/glamor_render.c
index 179b62dd8..7545268f3 100644
--- a/glamor/glamor_render.c
+++ b/glamor/glamor_render.c
@@ -838,6 +838,20 @@ glamor_render_format_is_supported(PicturePtr picture)
     }
 }
 
+static Bool
+render_op_uses_src_alpha(CARD8 op)
+{
+    struct blendinfo *info = &composite_op_info[op];
+
+    switch (info->dest_blend) {
+    case GL_ONE_MINUS_SRC_ALPHA:
+    case GL_SRC_ALPHA:
+        return TRUE;
+    }
+
+    return FALSE;
+}
+
 static Bool
 glamor_composite_choose_shader(CARD8 op,
                                PicturePtr source,
@@ -951,7 +965,8 @@ glamor_composite_choose_shader(CARD8 op,
         key.dest_swizzle = SHADER_DEST_SWIZZLE_ALPHA_TO_RED;
     } else {
         if (dest_pixmap->drawable.depth == 32 &&
-            glamor_drawable_effective_depth(dest->pDrawable) == 24)
+            glamor_drawable_effective_depth(dest->pDrawable) == 24 &&
+            !render_op_uses_src_alpha(op))
             key.dest_swizzle = SHADER_DEST_SWIZZLE_IGNORE_ALPHA;
         else
             key.dest_swizzle = SHADER_DEST_SWIZZLE_DEFAULT;
@@ -1185,6 +1200,7 @@ glamor_composite_with_shader(CARD8 op,
     Bool ret = FALSE;
     glamor_composite_shader *shader = NULL, *shader_ca = NULL;
     struct blendinfo op_info, op_info_ca;
+    Bool restore_colormask = FALSE;
 
     if (!glamor_composite_choose_shader(op, source, mask, dest,
                                         source_pixmap, mask_pixmap, dest_pixmap,
@@ -1209,6 +1225,14 @@ glamor_composite_with_shader(CARD8 op,
 
     glamor_make_current(glamor_priv);
 
+    if (ca_state != CA_TWO_PASS &&
+        key.dest_swizzle == SHADER_DEST_SWIZZLE_DEFAULT &&
+        dest_pixmap->drawable.depth == 32 &&
+        glamor_drawable_effective_depth(dest->pDrawable) == 24) {
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_FALSE);
+        restore_colormask = TRUE;
+    }
+
     glamor_set_destination_pixmap_priv_nc(glamor_priv, dest_pixmap, dest_pixmap_priv);
     glamor_composite_set_shader_blend(glamor_priv, dest_pixmap_priv, &key, shader, &op_info);
     glamor_set_alu(screen, GXcopy);
@@ -1351,6 +1375,8 @@ glamor_composite_with_shader(CARD8 op,
 
     glDisable(GL_SCISSOR_TEST);
 disable_va:
+    if (restore_colormask)
+        glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_POS);
     glDisableVertexAttribArray(GLAMOR_VERTEX_SOURCE);
     glDisableVertexAttribArray(GLAMOR_VERTEX_MASK);