From 079c5ccbcd07c5e8d51239b79dc3cfed46fef506 Mon Sep 17 00:00:00 2001 From: Olivier Fourdan Date: Thu, 16 Sep 2021 10:18:03 +0200 Subject: [PATCH] xwayland/shm: Avoid integer overflow on large pixmaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Xwayland's xwl_shm_create_pixmap() computes the size of the shared memory pool to create using a size_t, yet the Wayland protocol uses an integer for that size. If the pool size becomes larger than INT32_MAX, we end up asking Wayland to create a shared memory pool of negative size which in turn will raise a protocol error which terminates the Wayland connection, and therefore Xwayland. Avoid that issue early by return a NULL pixmap in that case, which will trigger a BadAlloc error, but leave Xwayland alive. Signed-off-by: Olivier Fourdan Reviewed-by: Jonas Ådahl --- hw/xwayland/xwayland-shm.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hw/xwayland/xwayland-shm.c b/hw/xwayland/xwayland-shm.c index cf7e97ca3..ff128316d 100644 --- a/hw/xwayland/xwayland-shm.c +++ b/hw/xwayland/xwayland-shm.c @@ -234,6 +234,15 @@ xwl_shm_create_pixmap(ScreenPtr screen, (width == 0 && height == 0) || depth < 15) return fbCreatePixmap(screen, width, height, depth, hint); + stride = PixmapBytePad(width, depth); + size = stride * height; + /* Size in the protocol is an integer, make sure we don't exceed + * INT32_MAX or else the Wayland compositor will raise an error and + * kill the Wayland connection! + */ + if (size > INT32_MAX) + return NULL; + pixmap = fbCreatePixmap(screen, 0, 0, depth, hint); if (!pixmap) return NULL; @@ -242,8 +251,6 @@ xwl_shm_create_pixmap(ScreenPtr screen, if (xwl_pixmap == NULL) goto err_destroy_pixmap; - stride = PixmapBytePad(width, depth); - size = stride * height; xwl_pixmap->buffer = NULL; xwl_pixmap->size = size; fd = os_create_anonymous_file(size);