From 39b2354bbcca3f9556d6a22fb80384f0d0f68783 Mon Sep 17 00:00:00 2001
From: "Enrico Weigelt, metux IT consult" <info@metux.net>
Date: Mon, 24 Feb 2025 13:57:51 +0100
Subject: [PATCH] xnest: protect from memory allocation failure

Signed-off-by: Enrico Weigelt, metux IT consult <info@metux.net>
---
 hw/xnest/Color.c    | 16 ++++++++++++++++
 hw/xnest/Display.c  |  2 ++
 hw/xnest/GC.c       |  2 ++
 hw/xnest/Keyboard.c |  3 ++-
 hw/xnest/Screen.c   |  2 ++
 5 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/hw/xnest/Color.c b/hw/xnest/Color.c
index 3ae507d5f..5f79a75f5 100644
--- a/hw/xnest/Color.c
+++ b/hw/xnest/Color.c
@@ -64,6 +64,8 @@ xnestCreateColormap(ColormapPtr pCmap)
     switch (pVisual->class) {
     case StaticGray:           /* read only */
         colors = calloc(ncolors, sizeof(XColor));
+        if (!colors)
+            return FALSE;
         for (i = 0; i < ncolors; i++)
             colors[i].pixel = i;
         XQueryColors(xnestDisplay, xnestColormap(pCmap), colors, ncolors);
@@ -77,6 +79,8 @@ xnestCreateColormap(ColormapPtr pCmap)
 
     case StaticColor:          /* read only */
         colors = calloc(ncolors, sizeof(XColor));
+        if (!colors)
+            return FALSE;
         for (i = 0; i < ncolors; i++)
             colors[i].pixel = i;
         XQueryColors(xnestDisplay, xnestColormap(pCmap), colors, ncolors);
@@ -90,6 +94,8 @@ xnestCreateColormap(ColormapPtr pCmap)
 
     case TrueColor:            /* read only */
         colors = calloc(ncolors, sizeof(XColor));
+        if (!colors)
+            return FALSE;
         red = green = blue = 0L;
         redInc = lowbit(pVisual->redMask);
         greenInc = lowbit(pVisual->greenMask);
@@ -196,11 +202,17 @@ xnestSetInstalledColormapWindows(ScreenPtr pScreen)
     int numWindows;
 
     icws.cmapIDs = calloc(pScreen->maxInstalledCmaps, sizeof(Colormap));
+    if (!icws.cmapIDs)
+        return;
     icws.numCmapIDs = xnestListInstalledColormaps(pScreen, icws.cmapIDs);
     icws.numWindows = 0;
     WalkTree(pScreen, xnestCountInstalledColormapWindows, (void *) &icws);
     if (icws.numWindows) {
         icws.windows = calloc(icws.numWindows + 1, sizeof(Window));
+        if (!icws.windows) {
+            free(icws.cmapIDs);
+            return;
+        }
         icws.index = 0;
         WalkTree(pScreen, xnestGetInstalledColormapWindows, (void *) &icws);
         icws.windows[icws.numWindows] = xnestDefaultWindows[pScreen->myNum];
@@ -220,6 +232,8 @@ xnestSetInstalledColormapWindows(ScreenPtr pScreen)
         {
             int i;
             Window64 *windows = calloc(numWindows, sizeof(Window64));
+            if (!windows)
+                return;
 
             for (i = 0; i < numWindows; ++i)
                 windows[i] = icws.windows[i];
@@ -392,6 +406,8 @@ xnestStoreColors(ColormapPtr pCmap, int nColors, xColorItem * pColors)
     {
         int i;
         XColor *pColors64 = calloc(nColors, sizeof(XColor));
+        if (!pColors64)
+            return;
 
         for (i = 0; i < nColors; ++i) {
             pColors64[i].pixel = pColors[i].pixel;
diff --git a/hw/xnest/Display.c b/hw/xnest/Display.c
index 5c49a61ba..8b77d2dd3 100644
--- a/hw/xnest/Display.c
+++ b/hw/xnest/Display.c
@@ -118,6 +118,8 @@ xnestOpenDisplay(int argc, char *argv[])
 
     xnestNumDefaultColormaps = xnestNumVisuals;
     xnestDefaultColormaps = calloc(xnestNumDefaultColormaps, sizeof(Colormap));
+    if (!xnestDefaultColormaps)
+        return;
     for (i = 0; i < xnestNumDefaultColormaps; i++)
         xnestDefaultColormaps[i] = XCreateColormap(xnestDisplay,
                                                    DefaultRootWindow
diff --git a/hw/xnest/GC.c b/hw/xnest/GC.c
index 85241aee9..04b948ef8 100644
--- a/hw/xnest/GC.c
+++ b/hw/xnest/GC.c
@@ -204,6 +204,8 @@ xnestChangeClip(GCPtr pGC, int type, void *pValue, int nRects)
     case CT_REGION:
         nRects = RegionNumRects((RegionPtr) pValue);
         pRects = calloc(nRects, sizeof(*pRects));
+        if (!pRects)
+            break;
         pBox = RegionRects((RegionPtr) pValue);
         for (i = nRects; i-- > 0;) {
             pRects[i].x = pBox[i].x1;
diff --git a/hw/xnest/Keyboard.c b/hw/xnest/Keyboard.c
index 127d6dfad..0ca003fc3 100644
--- a/hw/xnest/Keyboard.c
+++ b/hw/xnest/Keyboard.c
@@ -119,7 +119,8 @@ xnestKeyboardProc(DeviceIntPtr pDev, int onoff)
                                            max_keycode - min_keycode + 1,
                                            &mapWidth);
             len = (max_keycode - min_keycode + 1) * mapWidth;
-            keymap = calloc(len, sizeof(KeySym));
+            if (!(keymap = calloc(len, sizeof(KeySym))))
+                goto XkbError;
             for (i = 0; i < len; ++i)
                 keymap[i] = keymap64[i];
             XFree(keymap64);
diff --git a/hw/xnest/Screen.c b/hw/xnest/Screen.c
index 1f9aa4d34..38703b395 100644
--- a/hw/xnest/Screen.c
+++ b/hw/xnest/Screen.c
@@ -165,6 +165,8 @@ xnestOpenScreen(ScreenPtr pScreen, int argc, char *argv[])
         return FALSE;
 
     visuals = calloc(xnestNumVisuals, sizeof(VisualRec));
+    if (!visuals)
+        return FALSE;
     numVisuals = 0;
 
     depths = (DepthPtr) malloc(MAXDEPTH * sizeof(DepthRec));