diff --git a/Xi/xipassivegrab.c b/Xi/xipassivegrab.c
index eccec0ab8..8aba97781 100644
--- a/Xi/xipassivegrab.c
+++ b/Xi/xipassivegrab.c
@@ -63,7 +63,7 @@ SProcXIPassiveGrabDevice(ClientPtr client)
     swaps(&stuff->mask_len);
     swaps(&stuff->num_modifiers);
 
-    mods = (uint32_t *) &stuff[1];
+    mods = (uint32_t *) &stuff[1] + stuff->mask_len;
 
     for (i = 0; i < stuff->num_modifiers; i++, mods++) {
         swapl(mods);
diff --git a/test/xi2/protocol-xipassivegrabdevice.c b/test/xi2/protocol-xipassivegrabdevice.c
index 1e2341eb7..c747ddf03 100644
--- a/test/xi2/protocol-xipassivegrabdevice.c
+++ b/test/xi2/protocol-xipassivegrabdevice.c
@@ -137,6 +137,7 @@ request_XIPassiveGrabDevice(ClientPtr client, xXIPassiveGrabDeviceReq * req,
 {
     int rc;
     int local_modifiers;
+    int mask_len;
 
     rc = ProcXIPassiveGrabDevice(&client_request);
     assert(rc == error);
@@ -153,10 +154,11 @@ request_XIPassiveGrabDevice(ClientPtr client, xXIPassiveGrabDeviceReq * req,
     swaps(&req->deviceid);
     local_modifiers = req->num_modifiers;
     swaps(&req->num_modifiers);
+    mask_len = req->mask_len;
     swaps(&req->mask_len);
 
     while (local_modifiers--) {
-        CARD32 *mod = ((CARD32 *) (req + 1)) + local_modifiers;
+        CARD32 *mod = (CARD32 *) (req + 1) + mask_len + local_modifiers;
 
         swapl(mod);
     }
@@ -228,6 +230,11 @@ test_XIPassiveGrabDevice(void)
     request->detail = XIAnyButton;
     request_XIPassiveGrabDevice(&client_request, request, Success, 0);
 
+    /* Set a few random masks to make sure we handle modifiers correctly */
+    SetBit(mask, XI_ButtonPress);
+    SetBit(mask, XI_KeyPress);
+    SetBit(mask, XI_Enter);
+
     /* some modifiers */
     request->num_modifiers = N_MODS;
     request->length += N_MODS;