diff --git a/xkb/xkbAccessX.c b/xkb/xkbAccessX.c
index 21df85d94..963150294 100644
--- a/xkb/xkbAccessX.c
+++ b/xkb/xkbAccessX.c
@@ -723,23 +723,22 @@ ProcessPointerEvent(InternalEvent *ev, DeviceIntPtr mouse)
         changed |= XkbPointerButtonMask;
     }
     else if (event->type == ET_ButtonRelease) {
-        if (xkbi) {
-            xkbi->lockedPtrButtons &= ~(1 << (event->detail.key & 0x7));
+        if (IsMaster(dev)) {
+            DeviceIntPtr source;
+            int rc;
 
-            if (IsMaster(dev)) {
-                DeviceIntPtr source;
-                int rc;
-
-                rc = dixLookupDevice(&source, event->sourceid, serverClient,
-                                     DixWriteAccess);
-                if (rc != Success)
-                    ErrorF("[xkb] bad sourceid '%d' on button release event.\n",
-                           event->sourceid);
-                else if (!IsXTestDevice(source, GetMaster(dev, MASTER_POINTER)))
-                    XkbFakeDeviceButton(dev, FALSE, event->detail.key);
-            }
+            rc = dixLookupDevice(&source, event->sourceid, serverClient,
+                    DixWriteAccess);
+            if (rc != Success)
+                ErrorF("[xkb] bad sourceid '%d' on button release event.\n",
+                        event->sourceid);
+            else if (!IsXTestDevice(source, GetMaster(dev, MASTER_POINTER)))
+                XkbFakeDeviceButton(dev, FALSE, event->detail.key);
         }
 
+        if (xkbi)
+            xkbi->lockedPtrButtons &= ~(1 << (event->detail.key & 0x7));
+
         changed |= XkbPointerButtonMask;
     }