diff --git a/hw/xwayland/xwayland-input.c b/hw/xwayland/xwayland-input.c
index 35905829e..b137b816b 100644
--- a/hw/xwayland/xwayland-input.c
+++ b/hw/xwayland/xwayland-input.c
@@ -3246,6 +3246,15 @@ sprite_check_lost_focus(SpritePtr sprite, WindowPtr window)
     if (master->lastSlave != get_pointer_device(xwl_seat))
         return FALSE;
 
+    /* If we left the surface with a button down, it means the wayland compositor
+     * has grabbed the pointer so we will not get button release events from the
+     * compositor, so leave the window processing untouched, so that we do not
+     * end up with the wrong cursor, for example, when processing events once
+     * the pointer enters the X11 surface again.
+     */
+    if (master->button->buttonsDown)
+        return FALSE;
+
     if (xwl_seat->focus_window != NULL &&
         xwl_seat->cursor_confinement_window != NULL &&
         xwl_seat->focus_window != xwl_seat->cursor_confinement_window)