modesetting: Defer crtc gamma size upgrade to drmmode_setup_colormap

Rather than trying to create a gamma ramp array of the appropriate size in
drmmode_crtc_init when the GAMMA_LUT property should be used, just flag the crtc
as wanting to use the GAMMA_LUT property and then replace the gamma ramp later,
right before calling xf86HandleColormaps. This avoids a problem during initial
startup where xf86RandR12CreateObjects12 hard-codes a gamma ramp size of 256,
causing xf86RandR12CrtcSetGamma to read past the end of the DIX layer's RandR
gamma ramp array:

  PreInit
    drmmode_pre_init
      drmmode_crtc_init
        crtc->gamma_size = 1024

  ScreenInit
    xf86CrtcScreenInit
      xf86RandR12Init
        xf86RandR12Init12
          xf86RandR12CreateObjects12
            RRCrtcCreate
              randr_crtc->gammaSize = 0
          xf86RandR12InitGamma(pScrn, 256)
            RRCrtcGammaSetSize
              randr_crtc->gammaSize = 256
          xf86RandR12InitGamma
            xf86RandR12CrtcInitGamma
              RRCrtcGammaSet
                xf86RandR12CrtcSetGamma
                  // crtc->gamma_size is 1024 here, while randr_crtc->gammaRed
                  // is a 256-element array.
                  memcpy(crtc->gamma_red, randr_crtc->gammaRed, crtc->gamma_size * sizeof(crtc->gamma_red[0]));
    drmmode_setup_colormap
      xf86HandleColormaps
        xf86RandR12InitGamma
          RRCrtcGammaSetSize
            randr_crtc->gammaSize = 1024

Fixes: 245b9db0 - modesetting: Use GAMMA_LUT when available
Closes: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1126
Signed-off-by: Aaron Plattner <aplattner@nvidia.com>
Reviewed-by: Robert Morell <rmorell@nvidia.com>

hw/xfree86/drivers/modesetting/drmmode_display.c

@@ -2404,38 +2404,10 @@ drmmode_crtc_init(ScrnInfoPtr pScrn, drmmode_ptr drmmode, drmModeResPtr mode_res
     xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
                    "Allocated crtc nr. %d to this screen.\n", num);
 
-    /* If the GAMMA_LUT property is available, replace the server's default
+    drmmode_crtc->use_gamma_lut =
-     * gamma ramps with ones of the appropriate size. */
+        drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id &&
-    if (drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id) {
+        drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value &&
-        Bool try_gamma_lut =
+        xf86ReturnOptValBool(drmmode->Options, OPTION_USE_GAMMA_LUT, TRUE);
-            xf86ReturnOptValBool(drmmode->Options, OPTION_USE_GAMMA_LUT, TRUE);
-        uint64_t size = drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value;
-
-        if (try_gamma_lut && size != crtc->gamma_size) {
-            uint16_t *gamma = malloc(3 * size * sizeof(uint16_t));
-
-            if (gamma) {
-                free(crtc->gamma_red);
-
-                crtc->gamma_size = size;
-                crtc->gamma_red = gamma;
-                crtc->gamma_green = gamma + size;
-                crtc->gamma_blue = gamma + size * 2;
-
-                drmmode_crtc->use_gamma_lut = TRUE;
-
-                xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
-                               "Gamma ramp set to %ld entries on CRTC %d\n",
-                               size, num);
-            } else {
-                xf86DrvMsg(pScrn->scrnIndex, X_WARNING,
-                           "Failed to allocate memory for %ld gamma ramp "
-                           "entries on CRTC %d. Falling back to legacy "
-                           "%d-entry mode.\n",
-                           size, num, crtc->gamma_size);
-            }
-        }
-    }
 
     if (drmmode_crtc->use_gamma_lut &&
         drmmode_crtc->props[DRMMODE_CRTC_CTM].prop_id) {
@@ -3906,15 +3878,64 @@ drmmode_load_palette(ScrnInfoPtr pScrn, int numColors,
     }
 }
 
+static Bool
+drmmode_crtc_upgrade_lut(xf86CrtcPtr crtc, int num)
+{
+    drmmode_crtc_private_ptr drmmode_crtc = crtc->driver_private;
+    uint64_t size;
+
+    if (!drmmode_crtc->use_gamma_lut)
+        return TRUE;
+
+    assert(drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].prop_id);
+
+    size = drmmode_crtc->props[DRMMODE_CRTC_GAMMA_LUT_SIZE].value;
+
+    if (size != crtc->gamma_size) {
+        ScrnInfoPtr pScrn = crtc->scrn;
+        uint16_t *gamma = malloc(3 * size * sizeof(uint16_t));
+
+        if (gamma) {
+            free(crtc->gamma_red);
+
+            crtc->gamma_size = size;
+            crtc->gamma_red = gamma;
+            crtc->gamma_green = gamma + size;
+            crtc->gamma_blue = gamma + size * 2;
+
+            xf86DrvMsgVerb(pScrn->scrnIndex, X_INFO, MS_LOGLEVEL_DEBUG,
+                           "Gamma ramp set to %ld entries on CRTC %d\n",
+                           size, num);
+        } else {
+            xf86DrvMsg(pScrn->scrnIndex, X_ERROR,
+                       "Failed to allocate memory for %ld gamma ramp entries "
+                       "on CRTC %d.\n",
+                       size, num);
+            return FALSE;
+        }
+    }
+
+    return TRUE;
+}
+
 Bool
 drmmode_setup_colormap(ScreenPtr pScreen, ScrnInfoPtr pScrn)
 {
+    xf86CrtcConfigPtr xf86_config = XF86_CRTC_CONFIG_PTR(pScrn);
+    int i;
+
     xf86DrvMsg(pScrn->scrnIndex, X_INFO,
               "Initializing kms color map for depth %d, %d bpc.\n",
               pScrn->depth, pScrn->rgbBits);
     if (!miCreateDefColormap(pScreen))
         return FALSE;
 
+    /* If the GAMMA_LUT property is available, replace the server's default
+     * gamma ramps with ones of the appropriate size. */
+    for (i = 0; i < xf86_config->num_crtc; i++)
+        if (!drmmode_crtc_upgrade_lut(xf86_config->crtc[i], i))
+            return FALSE;
+
     /* Adapt color map size and depth to color depth of screen. */
     if (!xf86HandleColormaps(pScreen, 1 << pScrn->rgbBits, 10,
                              drmmode_load_palette, NULL,