miext: Move SyncShm FDs out of the way of clients

Applications may end up allocating a bunch of shmfence objects, each
of which uses a file descriptor, which must be kept open lest some
other client ask for a copy of it later on.

Lacking an API that can turn a memory mapping back into a file
descriptor, about the best we can do is push the file descriptors out
of the way of other X clients so that we don't run out of the ability
to accept new connections.

This uses fcntl F_GETFD to push the FD up above MAXCLIENTS.

Signed-off-by: Keith Packard <keithp@keithp.com>
Reviewed-by: Julien Cristau <jcristau@debian.org>

include/os.h

@@ -686,4 +686,7 @@ LogPrintMarkers(void);
 extern _X_EXPORT void
 xorg_backtrace(void);
 
+extern _X_EXPORT int
+os_move_fd(int fd);
+
 #endif                          /* OS_H */

miext/sync/misyncshm.c

@@ -32,6 +32,7 @@
 #include "pixmapstr.h"
 #include <sys/mman.h>
 #include <unistd.h>
+#include <fcntl.h>
 #include <X11/xshmfence.h>
 
 static DevPrivateKeyRec syncShmFencePrivateKey;
@@ -126,6 +127,7 @@ miSyncShmCreateFenceFromFd(ScreenPtr pScreen, SyncFence *pFence, int fd, Bool in
 
     miSyncInitFence(pScreen, pFence, initially_triggered);
 
+    fd = os_move_fd(fd);
     pPriv->fence = xshmfence_map_shm(fd);
     if (pPriv->fence) {
         pPriv->fd = fd;
@@ -145,6 +147,7 @@ miSyncShmGetFenceFd(ScreenPtr pScreen, SyncFence *pFence)
         pPriv->fd = xshmfence_alloc_shm();
         if (pPriv->fd < 0)
             return -1;
+        pPriv->fd = os_move_fd(pPriv->fd);
         pPriv->fence = xshmfence_map_shm(pPriv->fd);
         if (!pPriv->fence) {
             close (pPriv->fd);

os/utils.c

@@ -2071,3 +2071,27 @@ FormatUInt64Hex(uint64_t num, char *string)
 
     string[len] = '\0';
 }
+
+/* Move a file descriptor out of the way of our select mask; this
+ * is useful for file descriptors which will never appear in the
+ * select mask to avoid reducing the number of clients that can
+ * connect to the server
+ */
+int
+os_move_fd(int fd)
+{
+    int newfd;
+
+#ifdef F_DUPFD_CLOEXEC
+    newfd = fcntl(fd, F_DUPFD_CLOEXEC, MAXCLIENTS);
+#else
+    newfd = fcntl(fd, F_DUPFD, MAXCLIENTS);
+#endif
+    if (newfd < 0)
+        return fd;
+#ifndef F_DUPFD_CLOEXEC
+    fcntl(newfd, F_SETFD, FD_CLOEXEC);
+#endif
+    close(fd);
+    return newfd;
+}