diff --git a/hw/xwayland/xwayland-drm-lease.c b/hw/xwayland/xwayland-drm-lease.c
index 9d3b90f56..224ad4c73 100644
--- a/hw/xwayland/xwayland-drm-lease.c
+++ b/hw/xwayland/xwayland-drm-lease.c
@@ -44,7 +44,24 @@ xwl_randr_lease_cleanup_outputs(RRLeasePtr rrLease)
 
     for (i = 0; i < rrLease->numOutputs; ++i) {
         output = rrLease->outputs[i]->devPrivate;
-        output->lease = NULL;
+        if (output) {
+            output->lease = NULL;
+        }
+    }
+}
+
+static void
+xwl_randr_lease_free_outputs(RRLeasePtr rrLease)
+{
+    struct xwl_output *xwl_output;
+    int i;
+
+    for (i = 0; i < rrLease->numOutputs; ++i) {
+        xwl_output = rrLease->outputs[i]->devPrivate;
+        if (xwl_output && xwl_output->withdrawn_connector) {
+            rrLease->outputs[i]->devPrivate = NULL;
+            xwl_output_remove(xwl_output);
+        }
     }
 }
 
@@ -71,6 +88,9 @@ drm_lease_handle_finished(void *data,
         AttendClient(lease->client);
         xwl_randr_lease_cleanup_outputs(lease->rrLease);
     }
+
+    /* Free the xwl_outputs that have been withdrawn while lease-able */
+    xwl_randr_lease_free_outputs(lease->rrLease);
 }
 
 static struct wp_drm_lease_v1_listener drm_lease_listener = {
@@ -324,7 +344,15 @@ static void
 lease_connector_handle_withdrawn(void *data,
                                  struct wp_drm_lease_connector_v1 *wp_drm_lease_connector_v1)
 {
-    xwl_output_remove(data);
+    struct xwl_output *xwl_output = data;
+
+    xwl_output->withdrawn_connector = TRUE;
+
+    /* Not removing the xwl_output if currently leased with Wayland */
+    if (xwl_output->lease)
+        return;
+
+    xwl_output_remove(xwl_output);
 }
 
 static const struct wp_drm_lease_connector_v1_listener lease_connector_listener = {
diff --git a/hw/xwayland/xwayland-output.h b/hw/xwayland/xwayland-output.h
index 5e070f90d..0d36f459a 100644
--- a/hw/xwayland/xwayland-output.h
+++ b/hw/xwayland/xwayland-output.h
@@ -67,6 +67,7 @@ struct xwl_output {
     struct wp_drm_lease_connector_v1 *lease_connector;
     struct xwl_drm_lease *lease;
     struct xwl_drm_lease_device *lease_device;
+    Bool withdrawn_connector;
 };
 
 /* Per client per output emulated randr/vidmode resolution info. */