diff --git a/hw/xwayland/xwayland-glamor-eglstream.c b/hw/xwayland/xwayland-glamor-eglstream.c
index 2821f9a58..06cc1ad2a 100644
--- a/hw/xwayland/xwayland-glamor-eglstream.c
+++ b/hw/xwayland/xwayland-glamor-eglstream.c
@@ -84,10 +84,6 @@ struct xwl_pixmap {
     struct wl_buffer *buffer;
     struct xwl_screen *xwl_screen;
 
-    /* The stream and associated resources have their own lifetime separate
-     * from the pixmap's */
-    int refcount;
-
     EGLStreamKHR stream;
     EGLSurface surface;
 };
@@ -281,9 +277,6 @@ xwl_eglstream_unref_pixmap_stream(struct xwl_pixmap *xwl_pixmap)
 {
     struct xwl_screen *xwl_screen = xwl_pixmap->xwl_screen;
 
-    if (--xwl_pixmap->refcount >= 1)
-        return;
-
     /* If we're using this stream in the current egl context, unbind it so the
      * driver doesn't keep it around until the next eglMakeCurrent()
      * don't have to keep it around until something else changes the surface
@@ -341,13 +334,6 @@ xwl_eglstream_set_window_pixmap(WindowPtr window, PixmapPtr pixmap)
          * useless
          */
         pending->is_valid = FALSE;
-
-        /* The compositor may still be using the stream, so we can't destroy
-         * it yet. We'll only have a guarantee that the stream is safe to
-         * destroy once we receive the pending wl_display_sync() for this
-         * stream
-         */
-        pending->xwl_pixmap->refcount++;
     }
 
     xwl_screen->screen->SetWindowPixmap = xwl_eglstream->SetWindowPixmap;
@@ -481,10 +467,8 @@ xwl_eglstream_queue_pending_stream(struct xwl_screen *xwl_screen,
 static void
 xwl_eglstream_buffer_release_callback(void *data, struct wl_buffer *wl_buffer)
 {
-    struct xwl_pixmap *xwl_pixmap = xwl_pixmap_get(data);
-
-    xwl_pixmap_buffer_release_cb(data, wl_buffer);
-    xwl_eglstream_unref_pixmap_stream(xwl_pixmap);
+    /* drop the reference we took in post_damage, freeing if necessary */
+    dixDestroyPixmap(data, 0);
 }
 
 static const struct wl_buffer_listener xwl_eglstream_buffer_release_listener = {
@@ -510,7 +494,6 @@ xwl_eglstream_create_pending_stream(struct xwl_screen *xwl_screen,
     xwl_glamor_egl_make_current(xwl_screen);
 
     xwl_pixmap->xwl_screen = xwl_screen;
-    xwl_pixmap->refcount = 1;
     xwl_pixmap->stream = eglCreateStreamKHR(xwl_screen->egl_display, NULL);
     stream_fd = eglGetStreamFileDescriptorKHR(xwl_screen->egl_display,
                                               xwl_pixmap->stream);
@@ -624,9 +607,8 @@ xwl_glamor_eglstream_post_damage(struct xwl_window *xwl_window,
     glBindVertexArray(saved_vao);
     glBindTexture(GL_TEXTURE_2D, 0);
 
-    /* After this we will hand off the eglstream's wl_buffer to the
-     * compositor, which will own it until it sends a release() event. */
-    xwl_pixmap->refcount++;
+    /* hang onto the pixmap until the compositor has released it */
+    pixmap->refcnt++;
 }
 
 static void