glamor: Fallback to software rendering on GLSL link failure

Instead of thowing fatal error on GLSL link failure, fall back to software
rendering. This allows using Glamor on systems with limited hardware resources
(such as i915).

Part-of: <https://gitlab.freedesktop.org/xorg/xserver/-/merge_requests/1173>

glamor/glamor.c

@@ -369,35 +369,6 @@ fallback:
 }
 
 
-static Bool
-glamor_check_instruction_count(int gl_version)
-{
-    GLint max_native_alu_instructions;
-
-    /* Avoid using glamor if the reported instructions limit is too low,
-     * as this would cause glamor to fallback on sw due to large shaders
-     * which ends up being unbearably slow.
-     */
-    if (gl_version < 30) {
-        if (!epoxy_has_gl_extension("GL_ARB_fragment_program")) {
-            ErrorF("GL_ARB_fragment_program required\n");
-            return FALSE;
-        }
-
-        glGetProgramivARB(GL_FRAGMENT_PROGRAM_ARB,
-                          GL_MAX_PROGRAM_NATIVE_ALU_INSTRUCTIONS_ARB,
-                          &max_native_alu_instructions);
-        if (max_native_alu_instructions < GLAMOR_MIN_ALU_INSTRUCTIONS) {
-            LogMessage(X_WARNING,
-                       "glamor requires at least %d instructions (%d reported)\n",
-                       GLAMOR_MIN_ALU_INSTRUCTIONS, max_native_alu_instructions);
-            return FALSE;
-        }
-    }
-
-    return TRUE;
-}
-
 static void GLAPIENTRY
 glamor_debug_output_callback(GLenum source,
                              GLenum type,
@@ -728,9 +699,6 @@ glamor_init(ScreenPtr screen, unsigned int flags)
             goto fail;
         }
 
-        if (!glamor_check_instruction_count(gl_version))
-            goto fail;
-
         /* Glamor rendering assumes that platforms with GLSL 130+
          * have instanced arrays, but this is not always the case.
          * etnaviv offers GLSL 140 with OpenGL 2.1.
@@ -891,7 +859,16 @@ glamor_init(ScreenPtr screen, unsigned int flags)
     ps->Glyphs = glamor_composite_glyphs;
 
     glamor_init_vbo(screen);
-    glamor_init_gradient_shader(screen);
+
+    glamor_priv->enable_gradient_shader = TRUE;
+
+    if (!glamor_init_gradient_shader(screen)) {
+        LogMessage(X_WARNING,
+                   "glamor%d: Cannot initialize gradient shader, falling back to software rendering for gradients\n",
+                   screen->myNum);
+        glamor_priv->enable_gradient_shader = FALSE;
+    }
+
     glamor_pixmap_init(screen);
     glamor_sync_init(screen);
 

glamor/glamor_core.c

@@ -78,7 +78,7 @@ glamor_compile_glsl_prog(GLenum type, const char *source)
     return prog;
 }
 
-void
+Bool
 glamor_link_glsl_prog(ScreenPtr screen, GLint prog, const char *format, ...)
 {
     GLint ok;
@@ -106,8 +106,9 @@ glamor_link_glsl_prog(ScreenPtr screen, GLint prog, const char *format, ...)
 
         glGetProgramInfoLog(prog, size, NULL, info);
         ErrorF("Failed to link: %s\n", info);
-        FatalError("GLSL link failure\n");
+        return FALSE;
     }
+    return TRUE;
 }
 
 

glamor/glamor_gradient.c

@@ -177,7 +177,7 @@ _glamor_create_getcolor_fs_source(ScreenPtr screen, int stops_count,
     }
 }
 
-static void
+static Bool
 _glamor_create_radial_gradient_program(ScreenPtr screen, int stops_count,
                                        int dyn_gen)
 {
@@ -316,7 +316,7 @@ _glamor_create_radial_gradient_program(ScreenPtr screen, int stops_count,
 
     if ((glamor_priv->radial_max_nstops >= stops_count) && (dyn_gen)) {
         /* Very Good, not to generate again. */
-        return;
+        return TRUE;
     }
 
     glamor_make_current(glamor_priv);
@@ -353,7 +353,10 @@ _glamor_create_radial_gradient_program(ScreenPtr screen, int stops_count,
     glBindAttribLocation(gradient_prog, GLAMOR_VERTEX_POS, "v_position");
     glBindAttribLocation(gradient_prog, GLAMOR_VERTEX_SOURCE, "v_texcoord");
 
-    glamor_link_glsl_prog(screen, gradient_prog, "radial gradient");
+    if (!glamor_link_glsl_prog(screen, gradient_prog, "radial gradient")) {
+        glDeleteProgram(gradient_prog);
+        return FALSE;
+    }
 
     if (dyn_gen) {
         index = 2;
@@ -367,9 +370,11 @@ _glamor_create_radial_gradient_program(ScreenPtr screen, int stops_count,
     }
 
     glamor_priv->gradient_prog[SHADER_GRADIENT_RADIAL][index] = gradient_prog;
+
+    return TRUE;
 }
 
-static void
+static Bool
 _glamor_create_linear_gradient_program(ScreenPtr screen, int stops_count,
                                        int dyn_gen)
 {
@@ -500,7 +505,7 @@ _glamor_create_linear_gradient_program(ScreenPtr screen, int stops_count,
 
     if ((glamor_priv->linear_max_nstops >= stops_count) && (dyn_gen)) {
         /* Very Good, not to generate again. */
-        return;
+        return TRUE;
     }
 
     glamor_make_current(glamor_priv);
@@ -533,7 +538,10 @@ _glamor_create_linear_gradient_program(ScreenPtr screen, int stops_count,
     glBindAttribLocation(gradient_prog, GLAMOR_VERTEX_POS, "v_position");
     glBindAttribLocation(gradient_prog, GLAMOR_VERTEX_SOURCE, "v_texcoord");
 
-    glamor_link_glsl_prog(screen, gradient_prog, "linear gradient");
+    if (!glamor_link_glsl_prog(screen, gradient_prog, "linear gradient")) {
+        glDeleteProgram(gradient_prog);
+        return FALSE;
+    }
 
     if (dyn_gen) {
         index = 2;
@@ -547,9 +555,11 @@ _glamor_create_linear_gradient_program(ScreenPtr screen, int stops_count,
     }
 
     glamor_priv->gradient_prog[SHADER_GRADIENT_LINEAR][index] = gradient_prog;
+
+    return TRUE;
 }
 
-void
+Bool
 glamor_init_gradient_shader(ScreenPtr screen)
 {
     glamor_screen_private *glamor_priv;
@@ -564,11 +574,15 @@ glamor_init_gradient_shader(ScreenPtr screen)
     glamor_priv->linear_max_nstops = 0;
     glamor_priv->radial_max_nstops = 0;
 
-    _glamor_create_linear_gradient_program(screen, 0, 0);
+    if (!_glamor_create_linear_gradient_program(screen, 0, 0) ||
-    _glamor_create_linear_gradient_program(screen, LINEAR_LARGE_STOPS, 0);
+        !_glamor_create_linear_gradient_program(screen, LINEAR_LARGE_STOPS, 0))
+        return FALSE;
 
-    _glamor_create_radial_gradient_program(screen, 0, 0);
+    if (!_glamor_create_radial_gradient_program(screen, 0, 0) ||
-    _glamor_create_radial_gradient_program(screen, RADIAL_LARGE_STOPS, 0);
+        !_glamor_create_radial_gradient_program(screen, RADIAL_LARGE_STOPS, 0))
+        return FALSE;
+
+    return TRUE;
 }
 
 static void

glamor/glamor_priv.h

@@ -247,6 +247,7 @@ typedef struct glamor_screen_private {
     Bool can_copyplane;
     Bool use_gpu_shader4;
     int max_fbo_size;
+    Bool enable_gradient_shader;
 
     /**
      * Stores information about supported formats. Note, that this list contains all
@@ -654,7 +655,7 @@ Bool glamor_get_drawable_location(const DrawablePtr drawable);
 void glamor_get_drawable_deltas(DrawablePtr drawable, PixmapPtr pixmap,
                                 int *x, int *y);
 GLint glamor_compile_glsl_prog(GLenum type, const char *source);
-void glamor_link_glsl_prog(ScreenPtr screen, GLint prog,
+Bool glamor_link_glsl_prog(ScreenPtr screen, GLint prog,
                            const char *format, ...) _X_ATTRIBUTE_PRINTF(3,4);
 void glamor_get_color_4f_from_pixel(PixmapPtr pixmap,
                                     unsigned long fg_pixel, GLfloat *color);
@@ -710,7 +711,7 @@ void glamor_trapezoids(CARD8 op,
                        int ntrap, xTrapezoid *traps);
 
 /* glamor_gradient.c */
-void glamor_init_gradient_shader(ScreenPtr screen);
+Bool glamor_init_gradient_shader(ScreenPtr screen);
 PicturePtr glamor_generate_linear_gradient_picture(ScreenPtr screen,
                                                    PicturePtr src_picture,
                                                    int x_source, int y_source,
@@ -987,6 +988,4 @@ void glamor_xv_render(glamor_port_private *port_priv, int id);
 
 #include "glamor_font.h"
 
-#define GLAMOR_MIN_ALU_INSTRUCTIONS 128 /* Minimum required number of native ALU instructions */
-
 #endif                          /* GLAMOR_PRIV_H */

glamor/glamor_program.c

@@ -378,7 +378,8 @@ glamor_build_program(ScreenPtr          screen,
         glBindFragDataLocationIndexed(prog->prog, 0, 1, "color1");
     }
 
-    glamor_link_glsl_prog(screen, prog->prog, "%s_%s", prim->name, fill->name);
+    if (!glamor_link_glsl_prog(screen, prog->prog, "%s_%s", prim->name, fill->name))
+        goto fail;
 
     prog->matrix_uniform = glamor_get_uniform(prog, glamor_program_location_none, "v_matrix");
     prog->fg_uniform = glamor_get_uniform(prog, glamor_program_location_fg, "fg");

glamor/glamor_render.c

@@ -61,7 +61,7 @@ static struct blendinfo composite_op_info[] = {
 
 #define RepeatFix			10
 static GLuint
-glamor_create_composite_fs(glamor_screen_private *glamor_priv, struct shader_key *key)
+glamor_create_composite_fs(glamor_screen_private *glamor_priv, struct shader_key *key, Bool enable_rel_sampler)
 {
     const char *repeat_define =
         "#define RepeatNone               	      0\n"
@@ -131,6 +131,15 @@ glamor_create_composite_fs(glamor_screen_private *glamor_priv, struct shader_key
         "	}\n"
         "	return vec4(texture(tex_image, tex).rgb, 1.0);\n"
         "}\n";
+    const char *stub_rel_sampler =
+        " vec4 rel_sampler_rgba(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        "{\n"
+        "	return texture(tex_image, tex);\n"
+        "}\n"
+        " vec4 rel_sampler_rgbx(sampler2D tex_image, vec2 tex, vec4 wh, int repeat)\n"
+        "{\n"
+        "	return vec4(texture(tex_image, tex).rgb, 1.0);\n"
+        "}\n";
 
     const char *source_solid_fetch =
         "uniform vec4 source;\n"
@@ -327,7 +336,8 @@ glamor_create_composite_fs(glamor_screen_private *glamor_priv, struct shader_key
                 GLAMOR_DEFAULT_PRECISION
                 "%s%s%s%s%s%s%s%s", header, GLAMOR_COMPAT_DEFINES_FS,
                 repeat_define, relocate_texture,
-                rel_sampler, source_fetch, mask_fetch, dest_swizzle, in);
+                enable_rel_sampler ? rel_sampler : stub_rel_sampler,
+                source_fetch, mask_fetch, dest_swizzle, in);
 
     prog = glamor_compile_glsl_prog(GL_FRAGMENT_SHADER, source);
     free(source);
@@ -391,18 +401,21 @@ glamor_create_composite_shader(ScreenPtr screen, struct shader_key *key,
     GLuint vs, fs, prog;
     GLint source_sampler_uniform_location, mask_sampler_uniform_location;
     glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
+    Bool enable_rel_sampler = TRUE;
 
     glamor_make_current(glamor_priv);
     vs = glamor_create_composite_vs(glamor_priv, key);
     if (vs == 0)
         return;
-    fs = glamor_create_composite_fs(glamor_priv, key);
+    fs = glamor_create_composite_fs(glamor_priv, key, enable_rel_sampler);
     if (fs == 0)
         return;
 
     prog = glCreateProgram();
     glAttachShader(prog, vs);
     glAttachShader(prog, fs);
+    glDeleteShader(vs);
+    glDeleteShader(fs);
 
     glBindAttribLocation(prog, GLAMOR_VERTEX_POS, "v_position");
     glBindAttribLocation(prog, GLAMOR_VERTEX_SOURCE, "v_texcoord0");
@@ -412,7 +425,22 @@ glamor_create_composite_shader(ScreenPtr screen, struct shader_key *key,
         glBindFragDataLocationIndexed(prog, 0, 0, "color0");
         glBindFragDataLocationIndexed(prog, 0, 1, "color1");
     }
-    glamor_link_glsl_prog(screen, prog, "composite");
+
+    if (!glamor_link_glsl_prog(screen, prog, "composite")) {
+        /* Failed to link the shader, try again without rel_sampler. */
+        enable_rel_sampler = FALSE;
+        glDetachShader(prog, fs);
+        fs = glamor_create_composite_fs(glamor_priv, key, enable_rel_sampler);
+        if (fs == 0)
+            return;
+        glAttachShader(prog, fs);
+        glDeleteShader(fs);
+
+        if (!glamor_link_glsl_prog(screen, prog, "composite")) {
+            glDeleteProgram(prog);
+            return;
+        }
+    }
 
     shader->prog = prog;
 
@@ -1407,6 +1435,7 @@ glamor_convert_gradient_picture(ScreenPtr screen,
     int error;
     PictFormatPtr pFormat;
     PictFormatShort format;
+    glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
 
     if (source->pDrawable) {
         pFormat = source->pFormat;
@@ -1416,7 +1445,7 @@ glamor_convert_gradient_picture(ScreenPtr screen,
         pFormat = PictureMatchFormat(screen, 32, format);
     }
 
-    if (!source->pDrawable) {
+    if (glamor_priv->enable_gradient_shader && !source->pDrawable) {
         if (source->pSourcePict->type == SourcePictTypeLinear) {
             dst = glamor_generate_linear_gradient_picture(screen,
                                                           source, x_source,