From 4bb1f976d5397949be06510524bede5ae3bbfe0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Michel=20D=C3=A4nzer?= <mdaenzer@redhat.com>
Date: Thu, 20 Jul 2023 10:15:15 +0200
Subject: [PATCH] xwayland/glamor: Avoid implicit redirection with depth 32
 parent windows

glamor ensures that a depth 32 pixmap backing a depth 24 window contains
fully opaque alpha channel values for the window's pixels, so we can
allow this without implicit redirection, saving pixmap storage and
intermediate copies.

Second attempt, after fixing a few regressions from the first attempt.
---
 hw/xwayland/xwayland-glamor.c | 109 ++++++++++++++++++++++++++++++++++
 hw/xwayland/xwayland-screen.h |   2 +
 2 files changed, 111 insertions(+)

diff --git a/hw/xwayland/xwayland-glamor.c b/hw/xwayland/xwayland-glamor.c
index 9ac1aa8d4..c6aa8eb17 100644
--- a/hw/xwayland/xwayland-glamor.c
+++ b/hw/xwayland/xwayland-glamor.c
@@ -25,6 +25,8 @@
 
 #include <xwayland-config.h>
 
+#include <compositeext.h>
+
 #define MESA_EGL_NO_X11_HEADERS
 #define EGL_NO_X11
 #include <glamor_egl.h>
@@ -893,6 +895,108 @@ xwl_glamor_allow_commits(struct xwl_window *xwl_window)
         return TRUE;
 }
 
+static void
+xwl_avoid_implicit_redirect(WindowPtr window)
+{
+    ScreenPtr screen = window->drawable.pScreen;
+    WindowOptPtr parent_optional;
+    VisualPtr parent_visual = NULL;
+    VisualPtr window_visual = NULL;
+    DepthPtr depth32 = NULL;
+    int i;
+
+    if (!window->optional)
+        return;
+
+    parent_optional = FindWindowWithOptional(window)->optional;
+    if (window->optional == parent_optional ||
+        window->optional->visual == parent_optional->visual ||
+        CompositeIsImplicitRedirectException(screen, parent_optional->visual,
+                                             window->optional->visual))
+        return;
+
+    for (i = 0; i < screen->numDepths; i++) {
+        if (screen->allowedDepths[i].depth == 32) {
+            depth32 = &screen->allowedDepths[i];
+            break;
+        }
+    }
+
+    if (!depth32)
+        return;
+
+    for (i = 0; i < depth32->numVids; i++) {
+        XID argb_vid = depth32->vids[i];
+
+        if (argb_vid != parent_optional->visual)
+            continue;
+
+        if (!compIsAlternateVisual(screen, argb_vid))
+            break;
+
+        for (i = 0; i < screen->numVisuals; i++) {
+            if (screen->visuals[i].vid == argb_vid) {
+                parent_visual = &screen->visuals[i];
+                break;
+            }
+        }
+    }
+
+    if (!parent_visual)
+        return;
+
+    for (i = 0; i < screen->numVisuals; i++) {
+        if (screen->visuals[i].vid == window->optional->visual) {
+            window_visual = &screen->visuals[i];
+            break;
+        }
+    }
+
+    if ((window_visual->class != TrueColor &&
+         window_visual->class != DirectColor) ||
+        window_visual->redMask != parent_visual->redMask ||
+        window_visual->greenMask != parent_visual->greenMask ||
+        window_visual->blueMask != parent_visual->blueMask ||
+        window_visual->offsetRed != parent_visual->offsetRed ||
+        window_visual->offsetGreen != parent_visual->offsetGreen ||
+        window_visual->offsetBlue != parent_visual->offsetBlue)
+        return;
+
+    CompositeRegisterImplicitRedirectionException(screen, parent_visual->vid, window_visual->vid);
+}
+
+static Bool
+xwl_glamor_create_window(WindowPtr window)
+{
+    ScreenPtr screen = window->drawable.pScreen;
+    struct xwl_screen *xwl_screen = xwl_screen_get(screen);
+    Bool ret;
+
+    if (window->parent)
+        xwl_avoid_implicit_redirect(window);
+
+    screen->CreateWindow = xwl_screen->CreateWindow;
+    ret = (*screen->CreateWindow) (window);
+    xwl_screen->CreateWindow = screen->CreateWindow;
+    screen->CreateWindow = xwl_glamor_create_window;
+
+    return ret;
+}
+
+static void
+xwl_glamor_reparent_window(WindowPtr window, WindowPtr old_parent)
+{
+    ScreenPtr screen = window->drawable.pScreen;
+    struct xwl_screen *xwl_screen = xwl_screen_get(screen);
+
+    xwl_avoid_implicit_redirect(window);
+
+    screen->ReparentWindow = xwl_screen->ReparentWindow;
+    (*screen->ReparentWindow) (window, old_parent);
+    xwl_screen->ReparentWindow = screen->ReparentWindow;
+    screen->ReparentWindow = xwl_glamor_reparent_window;
+}
+
 static Bool
 xwl_glamor_create_screen_resources(ScreenPtr screen)
 {
@@ -907,6 +1011,11 @@ xwl_glamor_create_screen_resources(ScreenPtr screen)
     if (!ret)
         return ret;
 
+    xwl_screen->CreateWindow = screen->CreateWindow;
+    screen->CreateWindow = xwl_glamor_create_window;
+    xwl_screen->ReparentWindow = screen->ReparentWindow;
+    screen->ReparentWindow = xwl_glamor_reparent_window;
+
     if (xwl_screen->rootless) {
         screen->devPrivate =
             fbCreatePixmap(screen, 0, 0, screen->rootDepth, 0);
diff --git a/hw/xwayland/xwayland-screen.h b/hw/xwayland/xwayland-screen.h
index da97d7f1d..94033aabf 100644
--- a/hw/xwayland/xwayland-screen.h
+++ b/hw/xwayland/xwayland-screen.h
@@ -70,12 +70,14 @@ struct xwl_screen {
 
     CreateScreenResourcesProcPtr CreateScreenResources;
     CloseScreenProcPtr CloseScreen;
+    CreateWindowProcPtr CreateWindow;
     RealizeWindowProcPtr RealizeWindow;
     UnrealizeWindowProcPtr UnrealizeWindow;
     DestroyWindowProcPtr DestroyWindow;
     XYToWindowProcPtr XYToWindow;
     SetWindowPixmapProcPtr SetWindowPixmap;
     ChangeWindowAttributesProcPtr ChangeWindowAttributes;
+    ReparentWindowProcPtr ReparentWindow;
     ResizeWindowProcPtr ResizeWindow;
     MoveWindowProcPtr MoveWindow;