diff --git a/hw/xnest/Color.c b/hw/xnest/Color.c
index 0fcd4ee1d..3e13ec752 100644
--- a/hw/xnest/Color.c
+++ b/hw/xnest/Color.c
@@ -252,9 +252,11 @@ xnestSetInstalledColormapWindows(ScreenPtr pScreen)
                                         pScreen->defColormap, X11_RESTYPE_COLORMAP,
                                         serverClient, DixUseAccess);
 
-            XSetWindowColormap(xnestDisplay,
-                               xnestDefaultWindows[pScreen->myNum],
-                               xnestColormap(pCmap));
+            uint32_t cmap = xnestColormap(pCmap);
+            xcb_change_window_attributes(xnestUpstreamInfo.conn,
+                                         xnestDefaultWindows[pScreen->myNum],
+                                         XCB_CW_COLORMAP,
+                                         &cmap);
         }
 #endif                          /* DUMB_WINDOW_MANAGERS */
     }