Fix Xinerama's consolidated visual handling.

Formerly the code claimed it could only handle up to 256 visuals, which
was true.  Also true, but not explicitly stated, was that it could only
handle visuals with VID < 256.  If you have enough screens, and subsystems
that add lots of visuals, you can easily run off the end.  (Made worse
because we allocate visual IDs from the same pool as XIDs.)  If your app
then chooses a visual > 256, then the Xinerama code would throw BadMatch
on CreateColormap and your app wouldn't start.

With this change, PanoramiXVisualTable is gone.  Other subsystems that
were using it as a translation table between each screen's visuals now
use a PanoramiXTranslateVisual() helper.

Xext/panoramiX.c

@@ -81,9 +81,6 @@ static DepthPtr		PanoramiXDepths;
 static int		PanoramiXNumVisuals;
 static VisualPtr	PanoramiXVisuals;
 
-/* We support at most 256 visuals */
-_X_EXPORT XID		*PanoramiXVisualTable = NULL;
-
 _X_EXPORT unsigned long XRC_DRAWABLE;
 _X_EXPORT unsigned long XRT_WINDOW;
 _X_EXPORT unsigned long XRT_PIXMAP;
@@ -702,143 +699,133 @@ Bool PanoramiXCreateConnectionBlock(void)
     return TRUE;
 }
 
-extern
+/*
-void PanoramiXConsolidate(void)
+ * This isn't just memcmp(), bitsPerRGBValue is skipped.  markv made that
+ * change way back before xf86 4.0, but the comment for _why_ is a bit
+ * opaque, so I'm not going to question it for now.
+ *
+ * This is probably better done as a screen hook so DBE/EVI/GLX can add
+ * their own tests, and adding privates to VisualRec so they don't have to
+ * do their own back-mapping.
+ */
+static Bool
+VisualsEqual(VisualPtr a, VisualPtr b)
 {
-    int 	i, j, k;
+    return ((a->class == b->class) &&
-    VisualPtr   pVisual, pVisual2;
+	(a->ColormapEntries == b->ColormapEntries) &&
-    ScreenPtr   pScreen, pScreen2;
+	(a->nplanes == b->nplanes) &&
-    DepthPtr    pDepth, pDepth2;
+	(a->redMask == b->redMask) &&
+	(a->greenMask == b->greenMask) &&
+	(a->blueMask == b->blueMask) &&
+	(a->offsetRed == b->offsetRed) &&
+	(a->offsetGreen == b->offsetGreen) &&
+	(a->offsetBlue == b->offsetBlue));
+}
+
+static void
+PanoramiXMaybeAddDepth(DepthPtr pDepth)
+{
+    ScreenPtr pScreen;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	for (k = 0; k < pScreen->numDepths; k++) {
+	    if (pScreen->allowedDepths[k].depth == pDepth->depth) {
+		found = TRUE;
+		break;
+	    }
+	}
+    }
+
+    if (!found)
+	return;
+
+    j = PanoramiXNumDepths;
+    PanoramiXNumDepths++;
+    PanoramiXDepths = xrealloc(PanoramiXDepths,
+	    PanoramiXNumDepths * sizeof(DepthRec));
+    PanoramiXDepths[j].depth = pDepth->depth;
+    PanoramiXDepths[j].numVids = 0;
+    /* XXX suboptimal, should grow these dynamically */
+    if(pDepth->numVids)
+	PanoramiXDepths[j].vids = xalloc(sizeof(VisualID) * pDepth->numVids);
+    else
+	PanoramiXDepths[j].vids = NULL;
+}
+
+static void
+PanoramiXMaybeAddVisual(VisualPtr pVisual)
+{
+    ScreenPtr pScreen;
+    VisualPtr candidate = NULL;
+    int j, k;
+    Bool found = FALSE;
+
+    for (j = 1; j < PanoramiXNumScreens; j++) {
+	pScreen = screenInfo.screens[j];
+	found = FALSE;
+
+	candidate = pScreen->visuals;
+	for (k = 0; k < pScreen->numVisuals; k++) {
+	    candidate++;
+	    if (VisualsEqual(pVisual, candidate)
+#ifdef GLXPROXY
+		&& glxMatchVisual(screenInfo.screens[0], pVisual, pScreen)
+#endif
+		    ) {
+		found = TRUE;
+		break;
+	    }
+	}
+
+	if (!found)
+	    return;
+    }
+
+    /* found a matching visual on all screens, add it to the subset list */
+    j = PanoramiXNumVisuals;
+    PanoramiXNumVisuals++;
+    PanoramiXVisuals = xrealloc(PanoramiXVisuals,
+	    PanoramiXNumVisuals * sizeof(VisualRec));
+
+    memcpy(&PanoramiXVisuals[j], pVisual, sizeof(VisualRec));
+
+    for (k = 0; k < PanoramiXNumDepths; k++) {
+	if (PanoramiXDepths[k].depth == pVisual->nplanes) {
+	    PanoramiXDepths[k].vids[PanoramiXDepths[k].numVids] = pVisual->vid;
+	    PanoramiXDepths[k].numVids++;
+	    break;
+	}	
+    }   
+}
+
+extern void
+PanoramiXConsolidate(void)
+{
+    int 	i;
     PanoramiXRes *root, *defmap, *saver;
-    Bool        foundDepth, missingDepth;
+    ScreenPtr   pScreen = screenInfo.screens[0];
-
+    DepthPtr    pDepth = pScreen->allowedDepths;
-    if(!PanoramiXVisualTable)
+    VisualPtr   pVisual = pScreen->visuals;
-	PanoramiXVisualTable = xcalloc(256 * MAXSCREENS, sizeof(XID));
-
-    pScreen = screenInfo.screens[0];
-    pVisual = pScreen->visuals; 
-    pDepth  = pScreen->allowedDepths;
 
     PanoramiXNumDepths = 0;
-    PanoramiXDepths = xcalloc(pScreen->numDepths,sizeof(DepthRec));
     PanoramiXNumVisuals = 0;
-    PanoramiXVisuals = xcalloc(pScreen->numVisuals,sizeof(VisualRec));
 
-    for (i = 0; i < pScreen->numDepths; i++, pDepth++) {
+    for (i = 0; i < pScreen->numDepths; i++)
-        missingDepth = FALSE;
+	PanoramiXMaybeAddDepth(pDepth++);
-        for (j = 1; j < PanoramiXNumScreens; j++) {
-             pScreen2 = screenInfo.screens[j];
-             pDepth2 = pScreen2->allowedDepths;
 
-             foundDepth = FALSE;
+    for (i = 0; i < pScreen->numVisuals; i++)
-             for (k = 0; k < pScreen2->numDepths; k++, pDepth2++) {
+	PanoramiXMaybeAddVisual(pVisual++);
-                 if(pDepth2->depth == pDepth->depth) {
-                     foundDepth = TRUE;
-                     break;
-                 }
-             }
 
-             if(!foundDepth) {
+    root = xalloc(sizeof(PanoramiXRes));
-                missingDepth = TRUE;
-                break;
-             }
-        }
-          
-        if(!missingDepth) {
-            PanoramiXDepths[PanoramiXNumDepths].depth = pDepth->depth;
-            PanoramiXDepths[PanoramiXNumDepths].numVids = 0;
-            if(pDepth->numVids)
-                PanoramiXDepths[PanoramiXNumDepths].vids = 
-                      xalloc(sizeof(VisualID) * pDepth->numVids);      
-            else
-                PanoramiXDepths[PanoramiXNumDepths].vids = NULL;
-            PanoramiXNumDepths++;
-        }
-    }
-
-    for (i = 0; i < pScreen->numVisuals; i++, pVisual++) {
-	PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = pVisual->vid;
-
-	/* check if the visual exists on all screens */
-	for (j = 1; j < PanoramiXNumScreens; j++) {
-	    pScreen2 = screenInfo.screens[j];
-
-#ifdef GLXPROXY
-	    pVisual2 = glxMatchVisual(pScreen, pVisual, pScreen2);
-	    if (pVisual2) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-		    pVisual2->vid;
-		continue;
-	    } else if (glxMatchVisual(pScreen, pVisual, pScreen)) {
-		PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] = 0;
-		break;
-	    }
-#endif
-	    pVisual2 = pScreen2->visuals;
-
-	    for (k = 0; k < pScreen2->numVisuals; k++, pVisual2++) {
-		if ((pVisual->class == pVisual2->class) &&
-		    (pVisual->ColormapEntries == pVisual2->ColormapEntries) &&
-		    (pVisual->nplanes == pVisual2->nplanes) &&
-		    (pVisual->redMask == pVisual2->redMask) &&
-		    (pVisual->greenMask == pVisual2->greenMask) &&
-		    (pVisual->blueMask == pVisual2->blueMask) &&
-		    (pVisual->offsetRed == pVisual2->offsetRed) &&
-		    (pVisual->offsetGreen == pVisual2->offsetGreen) &&
-		    (pVisual->offsetBlue == pVisual2->offsetBlue))
-		{
-                /* We merely assign the first visual that matches.  OpenGL
-                   will need to get involved at some point if you want
-                   match GLX visuals */
-			PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j] =
-					 pVisual2->vid;
-			break;
-		}
-	    }
-	}
-	
-	/* if it doesn't exist on all screens we can't use it */
-	for (j = 0; j < PanoramiXNumScreens; j++) {
-	    if (!PanoramiXVisualTable[(pVisual->vid * MAXSCREENS) + j]) {
-		PanoramiXVisualTable[pVisual->vid * MAXSCREENS] = 0;
-		break;
-	    }
-	}
-
-	/* if it does, make sure it's in the list of supported depths and visuals */
-	if(PanoramiXVisualTable[pVisual->vid * MAXSCREENS]) {
-	    PanoramiXVisuals[PanoramiXNumVisuals].vid = pVisual->vid;
-	    PanoramiXVisuals[PanoramiXNumVisuals].class = pVisual->class;
-	    PanoramiXVisuals[PanoramiXNumVisuals].bitsPerRGBValue = pVisual->bitsPerRGBValue;
-	    PanoramiXVisuals[PanoramiXNumVisuals].ColormapEntries = pVisual->ColormapEntries;
-	    PanoramiXVisuals[PanoramiXNumVisuals].nplanes = pVisual->nplanes;
-	    PanoramiXVisuals[PanoramiXNumVisuals].redMask = pVisual->redMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].greenMask = pVisual->greenMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].blueMask = pVisual->blueMask;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetRed = pVisual->offsetRed;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetGreen = pVisual->offsetGreen;
-	    PanoramiXVisuals[PanoramiXNumVisuals].offsetBlue = pVisual->offsetBlue;
-	    PanoramiXNumVisuals++;	
-
-	    for (j = 0; j < PanoramiXNumDepths; j++) {
-	        if (PanoramiXDepths[j].depth == pVisual->nplanes) {
-		    PanoramiXDepths[j].vids[PanoramiXDepths[j].numVids] = pVisual->vid;
-		    PanoramiXDepths[j].numVids++;
-		    break;
-		}	
-	    }   
-	}
-    } 
-
-
-    root = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
     root->type = XRT_WINDOW;
-    defmap = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    defmap = xalloc(sizeof(PanoramiXRes));
     defmap->type = XRT_COLORMAP;
-    saver = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes));
+    saver = xalloc(sizeof(PanoramiXRes));
     saver->type = XRT_WINDOW;
 
-
     for (i =  0; i < PanoramiXNumScreens; i++) {
 	root->info[i].id = WindowTable[i]->drawable.id;
 	root->u.win.class = InputOutput;
@@ -854,6 +841,31 @@ void PanoramiXConsolidate(void)
     AddResource(defmap->info[0].id, XRT_COLORMAP, defmap);
 }
 
+_X_EXPORT VisualID
+PanoramiXTranslateVisualID(int screen, VisualID orig)
+{
+    VisualPtr pVisual = NULL;
+    int i, j;
+
+    for (i = 0; i < PanoramiXNumVisuals; i++) {
+	if (orig == PanoramiXVisuals[i].vid) {
+	    pVisual = &PanoramiXVisuals[i];
+	    break;
+	}
+    }
+
+    if (!pVisual)
+	return 0;
+
+    /* found the original, now translate it relative to the backend screen */
+    for (i = 0; i < PanoramiXNumScreens; i++)
+	for (j = 0; j < screenInfo.screens[i]->numVisuals; j++)
+	    if (VisualsEqual(pVisual, &screenInfo.screens[i]->visuals[j]))
+		return screenInfo.screens[i]->visuals[j].vid;
+    
+    return 0;
+}
+
 
 /*
  *	PanoramiXResetProc()

Xext/panoramiXprocs.c

@@ -150,7 +150,7 @@ int PanoramiXCreateWindow(ClientPtr client)
 	if (cmap)
 	    *((CARD32 *) &stuff[1] + cmap_offset) = cmap->info[j].id;
 	if ( orig_visual != CopyFromParent ) 
-	    stuff->visual = PanoramiXVisualTable[(orig_visual*MAXSCREENS) + j];
+	    stuff->visual = PanoramiXTranslateVisualID(j, orig_visual);
         result = (*SavedProcVector[X_CreateWindow])(client);
         if(result != Success) break;
     }
@@ -2077,9 +2077,6 @@ int PanoramiXCreateColormap(ClientPtr client)
 		client, stuff->window, XRT_WINDOW, DixReadAccess)))
 	return BadWindow;    
 
-    if(!stuff->visual || (stuff->visual > 255)) 
-	return BadValue;
-
     if(!(newCmap = (PanoramiXRes *) xalloc(sizeof(PanoramiXRes))))
         return BadAlloc;
 
@@ -2092,7 +2089,7 @@ int PanoramiXCreateColormap(ClientPtr client)
     FOR_NSCREENS_BACKWARD(j){
 	stuff->mid = newCmap->info[j].id;
 	stuff->window = win->info[j].id;
-	stuff->visual = PanoramiXVisualTable[(orig_visual * MAXSCREENS) + j];
+	stuff->visual = PanoramiXTranslateVisualID(j, orig_visual);
 	result = (* SavedProcVector[X_CreateColormap])(client);
 	if(result != Success) break;
     }

Xext/panoramiXsrv.h

@@ -12,8 +12,8 @@ extern int PanoramiXNumScreens;
 extern PanoramiXData *panoramiXdataPtr;
 extern int PanoramiXPixWidth;
 extern int PanoramiXPixHeight;
-extern XID *PanoramiXVisualTable;
 
+extern VisualID PanoramiXTranslateVisualID(int screen, VisualID orig);
 extern void PanoramiXConsolidate(void);
 extern Bool PanoramiXCreateConnectionBlock(void);
 extern PanoramiXRes * PanoramiXFindIDByScrnum(RESTYPE, XID, int);

Xext/saver.c

@@ -1346,8 +1346,7 @@ ProcScreenSaverSetAttributes (ClientPtr client)
              *((CARD32 *) &stuff[1] + cmap_offset) = cmap->info[i].id;
 
           if (orig_visual != CopyFromParent) 
-            stuff->visualID = 
+            stuff->visualID = PanoramiXTranslateVisualID(i, orig_visual);
-                     PanoramiXVisualTable[(orig_visual*MAXSCREENS) + i];
 
           status = ScreenSaverSetAttributes(client);
        }

hw/dmx/glxProxy/glxcmds.c

@@ -66,7 +66,6 @@
 
 #ifdef PANORAMIX
 #include "panoramiXsrv.h"
-extern XID *PanoramiXVisualTable;
 #endif
 
 extern __GLXFBConfig **__glXFBConfigs;
@@ -2824,14 +2823,8 @@ int __glXGetFBConfigs(__GLXclientState *cl, GLbyte *pc)
 #ifdef PANORAMIX
 	   else if (!noPanoramiXExtension) {
 	      /* convert the associated visualId to the panoramix one */
-              for (v=0; v<255; v++) {
+	      pFBConfig->associatedVisualId =
-		 if ( PanoramiXVisualTable[ v * MAXSCREENS + screen ] ==
+		  PanoramiXTranslateVisualID(screen, v);
-		      associatedVisualId ) {
-		    associatedVisualId = v;
-		    break;
-		 } 
-	      }
-	      pFBConfig->associatedVisualId = associatedVisualId;
 	   }
 #endif
 	}

hw/xfree86/loader/extsym.c

@@ -44,7 +44,6 @@ extern RESTYPE ShmSegType, ShmPixType;
 extern Bool noPanoramiXExtension;
 extern int PanoramiXNumScreens;
 extern PanoramiXData *panoramiXdataPtr;
-extern XID *PanoramiXVisualTable;
 extern unsigned long XRT_WINDOW;
 extern unsigned long XRT_PIXMAP;
 extern unsigned long XRT_GC;
@@ -69,7 +68,6 @@ _X_HIDDEN void *extLookupTab[] = {
     SYMFUNC(XineramaDeleteResource)
     SYMVAR(PanoramiXNumScreens)
     SYMVAR(panoramiXdataPtr)
-    SYMVAR(PanoramiXVisualTable)
     SYMVAR(XRT_WINDOW)
     SYMVAR(XRT_PIXMAP)
     SYMVAR(XRT_GC)