From d1f26c3e7566ebb6178a855b08b0f06dec730fb5 Mon Sep 17 00:00:00 2001
From: Olivier Fourdan <ofourdan@redhat.com>
Date: Thu, 13 Jan 2022 09:07:57 +0100
Subject: [PATCH] xwayland: Raise the FD limit to the max
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Xwayland may open a fair amount of file descriptors for passing Wayland
buffers, even more so when using the `wl_shm` either for the pointer
cursors or for when GLAMOR is not usable.

As a result, Xwayland may hit the (soft) limit of file descriptors
leading to a Wayland protocol error and the termination of Xwayland.

To mitigate that risk, raise the limit to the maximum (hard) limit of
file descriptors (unless of course the limit was set explicitly from the
command line with "-lf").

Note that for completeness, the Wayland compositor may have to do the
same, otherwise the limit might get reached on the compositor side as
well.

Signed-off-by: Olivier Fourdan <ofourdan@redhat.com>
Suggested-by: Simon Ser <contact@emersion.fr>
Acked-by: Michel Dänzer <mdaenzer@redhat.com>
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1283
---
 hw/xwayland/xwayland.c | 37 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 36 insertions(+), 1 deletion(-)

diff --git a/hw/xwayland/xwayland.c b/hw/xwayland/xwayland.c
index 21587dbb6..603e8be4f 100644
--- a/hw/xwayland/xwayland.c
+++ b/hw/xwayland/xwayland.c
@@ -25,7 +25,12 @@
 
 #include <xwayland-config.h>
 
+#if !defined(SYSV) && !defined(WIN32)
+#include <sys/resource.h>
+#endif
+
 #include <stdio.h>
+#include <errno.h>
 
 #include <X11/Xatom.h>
 #include <selection.h>
@@ -34,6 +39,7 @@
 #include <compositeext.h>
 #include <compint.h>
 #include <glx_extinit.h>
+#include <opaque.h>
 #include <os.h>
 #include <xserver_poll.h>
 #include <propertyst.h>
@@ -115,6 +121,33 @@ xwl_show_version(void)
 #endif
 }
 
+static void
+try_raising_nofile_limit(void)
+{
+#ifdef RLIMIT_NOFILE
+    struct rlimit rlim;
+
+    /* Only fiddle with the limit if not set explicitly from the command line */
+    if (limitNoFile >= 0)
+        return;
+
+    if (getrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+        ErrorF("Failed to get the current nofile limit: %s\n", strerror(errno));
+        return;
+    }
+
+    rlim.rlim_cur = rlim.rlim_max;
+
+    if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) {
+        ErrorF("Failed to set the current nofile limit: %s\n", strerror(errno));
+        return;
+    }
+
+    LogMessageVerb(X_INFO, 3, "Raising the file descriptors limit to %li\n",
+                   rlim.rlim_max);
+#endif
+}
+
 static void
 xwl_add_listen_fd(int argc, char *argv[], int i)
 {
@@ -268,9 +301,11 @@ InitOutput(ScreenInfo * screen_info, int argc, char **argv)
     screen_info->bitmapBitOrder = BITMAP_BIT_ORDER;
     screen_info->numPixmapFormats = ARRAY_SIZE(depths);
 
-    if (serverGeneration == 1)
+    if (serverGeneration == 1) {
+        try_raising_nofile_limit();
         LoadExtensionList(xwayland_extensions,
                           ARRAY_SIZE(xwayland_extensions), FALSE);
+    }
 
     wl_log_set_handler_client(xwl_log_handler);