xnest: protect from memory allocation failure

Signed-off-by: Enrico Weigelt, metux IT consult <info@metux.net>

hw/xnest/Color.c

@@ -62,7 +62,8 @@ xnestCreateColormap(ColormapPtr pCmap)
 
     switch (pVisual->class) {
     case StaticGray:           /* read only */
-        colors = calloc(ncolors, sizeof(XColor));
+        if (!(colors = calloc(ncolors, sizeof(XColor))))
+            return FALSE;
         for (i = 0; i < ncolors; i++)
             colors[i].pixel = i;
         XQueryColors(xnestDisplay, xnestColormap(pCmap), colors, ncolors);
@@ -75,7 +76,8 @@ xnestCreateColormap(ColormapPtr pCmap)
         break;
 
     case StaticColor:          /* read only */
-        colors = calloc(ncolors, sizeof(XColor));
+        if (!(colors = calloc(ncolors, sizeof(XColor))))
+            return FALSE;
         for (i = 0; i < ncolors; i++)
             colors[i].pixel = i;
         XQueryColors(xnestDisplay, xnestColormap(pCmap), colors, ncolors);
@@ -88,7 +90,8 @@ xnestCreateColormap(ColormapPtr pCmap)
         break;
 
     case TrueColor:            /* read only */
-        colors = calloc(ncolors, sizeof(XColor));
+        if (!(colors = calloc(ncolors, sizeof(XColor))))
+            return FALSE;
         red = green = blue = 0L;
         redInc = lowbit(pVisual->redMask);
         greenInc = lowbit(pVisual->greenMask);
@@ -194,12 +197,16 @@ xnestSetInstalledColormapWindows(ScreenPtr pScreen)
     xnestInstalledColormapWindows icws;
     int numWindows;
 
-    icws.cmapIDs = calloc(pScreen->maxInstalledCmaps, sizeof(Colormap));
+    if (!(icws.cmapIDs = calloc(pScreen->maxInstalledCmaps, sizeof(Colormap))))
+        return;
     icws.numCmapIDs = xnestListInstalledColormaps(pScreen, icws.cmapIDs);
     icws.numWindows = 0;
     WalkTree(pScreen, xnestCountInstalledColormapWindows, (void *) &icws);
     if (icws.numWindows) {
-        icws.windows = calloc(icws.numWindows + 1, sizeof(Window));
+        if (!(icws.windows = calloc(icws.numWindows + 1, sizeof(Window)))) {
+            free(icws.cmapIDs);
+            return;
+        }
         icws.index = 0;
         WalkTree(pScreen, xnestGetInstalledColormapWindows, (void *) &icws);
         icws.windows[icws.numWindows] = xnestDefaultWindows[pScreen->myNum];
@@ -219,6 +226,8 @@ xnestSetInstalledColormapWindows(ScreenPtr pScreen)
         {
             int i;
             Window64 *windows = calloc(numWindows, sizeof(Window64));
+            if (!windows)
+                return;
 
             for (i = 0; i < numWindows; ++i)
                 windows[i] = icws.windows[i];
@@ -391,6 +400,8 @@ xnestStoreColors(ColormapPtr pCmap, int nColors, xColorItem * pColors)
     {
         int i;
         XColor *pColors64 = calloc(nColors, sizeof(XColor));
+        if (!pColors64)
+            return;
 
         for (i = 0; i < nColors; ++i) {
             pColors64[i].pixel = pColors[i].pixel;

hw/xnest/Display.c

@@ -117,7 +117,8 @@ xnestOpenDisplay(int argc, char *argv[])
     }
 
     xnestNumDefaultColormaps = xnestNumVisuals;
-    xnestDefaultColormaps = calloc(xnestNumDefaultColormaps, sizeof(Colormap));
+    if (!(xnestDefaultColormaps = calloc(xnestNumDefaultColormaps, sizeof(Colormap))))
+        return;
     for (i = 0; i < xnestNumDefaultColormaps; i++)
         xnestDefaultColormaps[i] = XCreateColormap(xnestDisplay,
                                                    DefaultRootWindow

hw/xnest/GC.c

@@ -203,7 +203,8 @@ xnestChangeClip(GCPtr pGC, int type, void *pValue, int nRects)
 
     case CT_REGION:
         nRects = RegionNumRects((RegionPtr) pValue);
-        pRects = calloc(nRects, sizeof(*pRects));
+        if (!(pRects = calloc(nRects, sizeof(*pRects))))
+            break;
         pBox = RegionRects((RegionPtr) pValue);
         for (i = nRects; i-- > 0;) {
             pRects[i].x = pBox[i].x1;

hw/xnest/Keyboard.c

@@ -119,7 +119,8 @@ xnestKeyboardProc(DeviceIntPtr pDev, int onoff)
                                            max_keycode - min_keycode + 1,
                                            &mapWidth);
             len = (max_keycode - min_keycode + 1) * mapWidth;
-            keymap = calloc(len, sizeof(KeySym));
+            if (!(keymap = calloc(len, sizeof(KeySym))))
+                goto XkbError;
             for (i = 0; i < len; ++i)
                 keymap[i] = keymap64[i];
             XFree(keymap64);

hw/xnest/Screen.c

@@ -163,7 +163,8 @@ xnestOpenScreen(ScreenPtr pScreen, int argc, char *argv[])
                                      PRIVATE_CURSOR, 0))
         return FALSE;
 
-    visuals = calloc(xnestNumVisuals, sizeof(VisualRec));
+    if (!(visuals = calloc(xnestNumVisuals, sizeof(VisualRec))))
+        return FALSE;
     numVisuals = 0;
 
     depths = calloc(MAXDEPTH, sizeof(DepthRec));