modesetting: Disable pageflipping when using a swcursor

The miPointerSpriteFunc swcursor code expects there to only be a single
framebuffer and when the cursor moves it will undo the damage of the
previous draw, potentially overwriting what ever is there in a new
framebuffer installed after a flip.

This leads to all kind of artifacts, so we need to disable pageflipping
when a swcursor is used.

The code for this has shamelessly been copied from the xf86-video-amdgpu
code.

Fixes: https://gitlab.freedesktop.org/xorg/xserver/issues/828

Reviewed-by: Michel Dänzer <michel.daenzer@amd.com>
Signed-off-by: Hans de Goede <hdegoede@redhat.com>
(cherry picked from commit 0aaac8d783)
Signed-off-by: Łukasz Spintzyk <lukasz.spintzyk@displaylink.com>

hw/xfree86/drivers/modesetting/dri2.c

@@ -586,6 +586,7 @@ can_flip(ScrnInfoPtr scrn, DrawablePtr draw,
 
     return draw->type == DRAWABLE_WINDOW &&
         ms->drmmode.pageflip &&
+        !ms->drmmode.sprites_visible &&
         !ms->drmmode.present_flipping &&
         scrn->vtSema &&
         DRI2CanFlip(draw) && can_exchange(scrn, draw, front, back);

hw/xfree86/drivers/modesetting/driver.c

@@ -41,6 +41,7 @@
 #include "compiler.h"
 #include "xf86Pci.h"
 #include "mipointer.h"
+#include "mipointrst.h"
 #include "micmap.h"
 #include <X11/extensions/randr.h>
 #include "fb.h"
@@ -1656,6 +1657,21 @@ ScreenInit(ScreenPtr pScreen, int argc, char **argv)
     xf86SetSilkenMouse(pScreen);
     miDCInitialize(pScreen, xf86GetPointerScreenFuncs());
 
+    /* If pageflip is enabled hook the screen's cursor-sprite (swcursor) funcs.
+     * So that we can disabe page-flipping on fallback to a swcursor. */
+    if (ms->drmmode.pageflip) {
+        miPointerScreenPtr PointPriv =
+            dixLookupPrivate(&pScreen->devPrivates, miPointerScreenKey);
+
+        if (!dixRegisterScreenPrivateKey(&ms->drmmode.spritePrivateKeyRec,
+                                         pScreen, PRIVATE_DEVICE,
+                                         sizeof(msSpritePrivRec)))
+            return FALSE;
+
+        ms->SpriteFuncs = PointPriv->spriteFuncs;
+        PointPriv->spriteFuncs = &drmmode_sprite_funcs;
+    }
+
     /* Need to extend HWcursor support to handle mask interleave */
     if (!ms->drmmode.sw_cursor)
         xf86_cursors_init(pScreen, ms->cursor_width, ms->cursor_height,
@@ -1852,6 +1868,14 @@ CloseScreen(ScreenPtr pScreen)
 
     drmmode_free_bos(pScrn, &ms->drmmode);
 
+    if (ms->drmmode.pageflip) {
+        miPointerScreenPtr PointPriv =
+            dixLookupPrivate(&pScreen->devPrivates, miPointerScreenKey);
+
+        if (PointPriv->spriteFuncs == &drmmode_sprite_funcs)
+            PointPriv->spriteFuncs = ms->SpriteFuncs;        
+    }
+
     if (pScrn->vtSema) {
         LeaveVT(pScrn);
     }

hw/xfree86/drivers/modesetting/driver.h

@@ -97,6 +97,7 @@ typedef struct _modesettingRec {
 
     CreateScreenResourcesProcPtr createScreenResources;
     ScreenBlockHandlerProcPtr BlockHandler;
+    miPointerSpriteFuncPtr SpriteFuncs;
     void *driver;
 
     drmmode_rec drmmode;

hw/xfree86/drivers/modesetting/drmmode_display.c

@@ -34,6 +34,7 @@
 #include <sys/mman.h>
 #include <unistd.h>
 #include "dumb_bo.h"
+#include "inputstr.h"
 #include "xf86str.h"
 #include "X11/Xatom.h"
 #include "mi.h"
@@ -3938,3 +3939,102 @@ drmmode_get_default_bpp(ScrnInfoPtr pScrn, drmmode_ptr drmmode, int *depth,
     drmModeFreeResources(mode_res);
     return;
 }
+
+/*
+ * We hook the screen's cursor-sprite (swcursor) functions to see if a swcursor
+ * is active. When a swcursor is active we disabe page-flipping.
+ */
+
+static void drmmode_sprite_do_set_cursor(msSpritePrivPtr sprite_priv,
+                                         ScrnInfoPtr scrn, int x, int y)
+{
+    modesettingPtr ms = modesettingPTR(scrn);
+    CursorPtr cursor = sprite_priv->cursor;
+    Bool sprite_visible = sprite_priv->sprite_visible;
+
+    if (cursor) {
+        x -= cursor->bits->xhot;
+        y -= cursor->bits->yhot;
+
+        sprite_priv->sprite_visible =
+            x < scrn->virtualX && y < scrn->virtualY &&
+            (x + cursor->bits->width > 0) &&
+            (y + cursor->bits->height > 0);
+    } else {
+        sprite_priv->sprite_visible = FALSE;
+    }
+
+    ms->drmmode.sprites_visible += sprite_priv->sprite_visible - sprite_visible;
+}
+
+static void drmmode_sprite_set_cursor(DeviceIntPtr pDev, ScreenPtr pScreen,
+                                      CursorPtr pCursor, int x, int y)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+    msSpritePrivPtr sprite_priv = msGetSpritePriv(pDev, ms, pScreen);
+
+    sprite_priv->cursor = pCursor;
+    drmmode_sprite_do_set_cursor(sprite_priv, scrn, x, y);
+
+    ms->SpriteFuncs->SetCursor(pDev, pScreen, pCursor, x, y);
+}
+
+static void drmmode_sprite_move_cursor(DeviceIntPtr pDev, ScreenPtr pScreen,
+                                       int x, int y)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+    msSpritePrivPtr sprite_priv = msGetSpritePriv(pDev, ms, pScreen);
+
+    drmmode_sprite_do_set_cursor(sprite_priv, scrn, x, y);
+
+    ms->SpriteFuncs->MoveCursor(pDev, pScreen, x, y);
+}
+
+static Bool drmmode_sprite_realize_realize_cursor(DeviceIntPtr pDev,
+                                                  ScreenPtr pScreen,
+                                                  CursorPtr pCursor)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+
+    return ms->SpriteFuncs->RealizeCursor(pDev, pScreen, pCursor);
+}
+
+static Bool drmmode_sprite_realize_unrealize_cursor(DeviceIntPtr pDev,
+                                                    ScreenPtr pScreen,
+                                                    CursorPtr pCursor)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+
+    return ms->SpriteFuncs->UnrealizeCursor(pDev, pScreen, pCursor);
+}
+
+static Bool drmmode_sprite_device_cursor_initialize(DeviceIntPtr pDev,
+                                                    ScreenPtr pScreen)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+
+    return ms->SpriteFuncs->DeviceCursorInitialize(pDev, pScreen);
+}
+
+static void drmmode_sprite_device_cursor_cleanup(DeviceIntPtr pDev,
+                                                 ScreenPtr pScreen)
+{
+    ScrnInfoPtr scrn = xf86ScreenToScrn(pScreen);
+    modesettingPtr ms = modesettingPTR(scrn);
+
+    ms->SpriteFuncs->DeviceCursorCleanup(pDev, pScreen);
+}
+
+miPointerSpriteFuncRec drmmode_sprite_funcs = {
+    .RealizeCursor = drmmode_sprite_realize_realize_cursor,
+    .UnrealizeCursor = drmmode_sprite_realize_unrealize_cursor,
+    .SetCursor = drmmode_sprite_set_cursor,
+    .MoveCursor = drmmode_sprite_move_cursor,
+    .DeviceCursorInitialize = drmmode_sprite_device_cursor_initialize,
+    .DeviceCursorCleanup = drmmode_sprite_device_cursor_cleanup,
+};

hw/xfree86/drivers/modesetting/drmmode_display.h

@@ -111,6 +111,9 @@ typedef struct {
     void *shadow_fb2;
 
     DevPrivateKeyRec pixmapPrivateKeyRec;
+    DevScreenPrivateKeyRec spritePrivateKeyRec;
+    /* Number of SW cursors currently visible on this screen */
+    int sprites_visible;
 
     Bool reverse_prime_offload_mode;
 
@@ -242,6 +245,15 @@ extern DevPrivateKeyRec msPixmapPrivateKeyRec;
 
 #define msGetPixmapPriv(drmmode, p) ((msPixmapPrivPtr)dixGetPrivateAddr(&(p)->devPrivates, &(drmmode)->pixmapPrivateKeyRec))
 
+typedef struct _msSpritePriv {
+    CursorPtr cursor;
+    Bool sprite_visible;
+} msSpritePrivRec, *msSpritePrivPtr;
+
+#define msGetSpritePriv(dev, ms, screen) dixLookupScreenPrivate(&(dev)->devPrivates, &(ms)->drmmode.spritePrivateKeyRec, screen)
+
+extern miPointerSpriteFuncRec drmmode_sprite_funcs;
+
 Bool drmmode_is_format_supported(ScrnInfoPtr scrn, uint32_t format,
                                  uint64_t modifier);
 int drmmode_bo_import(drmmode_ptr drmmode, drmmode_bo *bo,

hw/xfree86/drivers/modesetting/present.c

@@ -208,9 +208,13 @@ ms_present_flip_abort(modesettingPtr ms, void *data)
 
 /*
  * Test to see if page flipping is possible on the target crtc
+ *
+ * We ignore sw-cursors when *disabling* flipping, we may very well be
+ * returning to scanning out the normal framebuffer *because* we just
+ * switched to sw-cursor mode and check_flip just failed because of that.
  */
 static Bool
-ms_present_check_flip(RRCrtcPtr crtc,
+ms_present_check_unflip(RRCrtcPtr crtc,
                         WindowPtr window,
                         PixmapPtr pixmap,
                         Bool sync_flip,
@@ -281,6 +285,23 @@ ms_present_check_flip(RRCrtcPtr crtc,
     return TRUE;
 }
 
+static Bool
+ms_present_check_flip(RRCrtcPtr crtc,
+                      WindowPtr window,
+                      PixmapPtr pixmap,
+                      Bool sync_flip,
+                      PresentFlipReason *reason)
+{
+    ScreenPtr screen = window->drawable.pScreen;
+    ScrnInfoPtr scrn = xf86ScreenToScrn(screen);
+    modesettingPtr ms = modesettingPTR(scrn);
+
+    if (ms->drmmode.sprites_visible > 0)
+        return FALSE;
+
+    return ms_present_check_unflip(crtc, window, pixmap, sync_flip, reason);
+}
+
 /*
  * Queue a flip on 'crtc' to 'pixmap' at 'target_msc'. If 'sync_flip' is true,
  * then wait for vblank. Otherwise, flip immediately
@@ -343,7 +364,7 @@ ms_present_unflip(ScreenPtr screen, uint64_t event_id)
     event->event_id = event_id;
     event->unflip = TRUE;
 
-    if (ms_present_check_flip(NULL, screen->root, pixmap, TRUE, NULL) &&
+    if (ms_present_check_unflip(NULL, screen->root, pixmap, TRUE, NULL) &&
         ms_do_pageflip(screen, pixmap, event, -1, FALSE,
                        ms_present_flip_handler, ms_present_flip_abort)) {
         return;