From de959ec939b262cb1cb4c0b6146826e3092843f9 Mon Sep 17 00:00:00 2001
From: Eric Anholt <eric@anholt.net>
Date: Thu, 5 Nov 2015 14:47:42 -0800
Subject: [PATCH] glamor: Handle GL_OUT_OF_MEMORY when allocating texture
 images.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The spec allows general undefined behavior when GL_OOM is thrown.  But
if the driver happens to throw the error at this point, it probably
means the pixmap was just too big, so we should delete that texture
and have this pixmap fall back to software.

Signed-off-by: Eric Anholt <eric@anholt.net>
Reviewed-by: Michel Dänzer <michel.daenzer@amd.com>
---
 glamor/glamor.c      |  7 +++++++
 glamor/glamor_fbo.c  | 18 ++++++++++++++++++
 glamor/glamor_priv.h |  3 +++
 3 files changed, 28 insertions(+)

diff --git a/glamor/glamor.c b/glamor/glamor.c
index d4a023629..e69f83d8b 100644
--- a/glamor/glamor.c
+++ b/glamor/glamor.c
@@ -379,6 +379,13 @@ glamor_debug_output_callback(GLenum source,
                              const void *userParam)
 {
     ScreenPtr screen = (void *)userParam;
+    glamor_screen_private *glamor_priv = glamor_get_screen_private(screen);
+
+    if (glamor_priv->suppress_gl_out_of_memory_logging &&
+        source == GL_DEBUG_SOURCE_API && type == GL_DEBUG_TYPE_ERROR) {
+        return;
+    }
+
     LogMessageVerb(X_ERROR, 0, "glamor%d: GL error: %*s\n",
                screen->myNum, length, message);
 }
diff --git a/glamor/glamor_fbo.c b/glamor/glamor_fbo.c
index 262033f6b..545f89fcd 100644
--- a/glamor/glamor_fbo.c
+++ b/glamor/glamor_fbo.c
@@ -347,9 +347,25 @@ _glamor_create_tex(glamor_screen_private *glamor_priv,
         glBindTexture(GL_TEXTURE_2D, tex);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
         glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+
+        glamor_priv->suppress_gl_out_of_memory_logging = true;
         glTexImage2D(GL_TEXTURE_2D, 0, format, w, h, 0,
                      format, GL_UNSIGNED_BYTE, NULL);
+        glamor_priv->suppress_gl_out_of_memory_logging = false;
     }
+
+    if (glGetError() == GL_OUT_OF_MEMORY) {
+        if (!glamor_priv->logged_any_fbo_allocation_failure) {
+            LogMessageVerb(X_WARNING, 0, "glamor: Failed to allocate %dx%d "
+                           "FBO due to GL_OUT_OF_MEMORY.\n", w, h);
+            LogMessageVerb(X_WARNING, 0,
+                           "glamor: Expect reduced performance.\n");
+            glamor_priv->logged_any_fbo_allocation_failure = true;
+        }
+        glDeleteTextures(1, &tex);
+        return 0;
+    }
+
     return tex;
 }
 
@@ -368,6 +384,8 @@ glamor_create_fbo(glamor_screen_private *glamor_priv,
         return fbo;
  new_fbo:
     tex = _glamor_create_tex(glamor_priv, w, h, format, flag == CREATE_PIXMAP_USAGE_SHARED);
+    if (!tex)
+        return NULL;
     fbo = glamor_create_fbo_from_tex(glamor_priv, w, h, format, tex, flag);
 
     return fbo;
diff --git a/glamor/glamor_priv.h b/glamor/glamor_priv.h
index f89d56d9e..a190e67b1 100644
--- a/glamor/glamor_priv.h
+++ b/glamor/glamor_priv.h
@@ -293,6 +293,9 @@ typedef struct glamor_screen_private {
     ScreenPtr screen;
     int dri3_enabled;
 
+    Bool suppress_gl_out_of_memory_logging;
+    Bool logged_any_fbo_allocation_failure;
+
     /* xv */
     GLint xv_prog;