Merge pull request #5856 from stenzek/optimus-crash

Fix ubershader crashes with primus/bumblebee

Source/Core/Common/GL/GLInterface/GLX.cpp

@@ -148,9 +148,9 @@ bool cInterfaceGLX::Create(void* window_handle, bool stereo, bool core)
   if (!ctx || s_glxError)
   {
     ERROR_LOG(VIDEO, "Unable to create GL context.");
+    XSetErrorHandler(oldHandler);
     return false;
   }
-  XSetErrorHandler(oldHandler);
 
   std::string tmp;
   std::istringstream buffer(glXQueryExtensionsString(dpy, screen));
@@ -172,8 +172,11 @@ bool cInterfaceGLX::Create(void* window_handle, bool stereo, bool core)
   if (!CreateWindowSurface())
   {
     ERROR_LOG(VIDEO, "Error: CreateWindowSurface failed\n");
+    XSetErrorHandler(oldHandler);
     return false;
   }
+
+  XSetErrorHandler(oldHandler);
   return true;
 }
 
@@ -182,6 +185,7 @@ bool cInterfaceGLX::Create(cInterfaceBase* main_context)
   cInterfaceGLX* glx_context = static_cast<cInterfaceGLX*>(main_context);
 
   m_has_handle = false;
+  m_supports_pbuffer = glx_context->m_supports_pbuffer;
   dpy = glx_context->dpy;
   fbconfig = glx_context->fbconfig;
   s_glxError = false;
@@ -193,15 +197,18 @@ bool cInterfaceGLX::Create(cInterfaceBase* main_context)
   if (!ctx || s_glxError)
   {
     ERROR_LOG(VIDEO, "Unable to create GL context.");
+    XSetErrorHandler(oldHandler);
     return false;
   }
-  XSetErrorHandler(oldHandler);
 
-  if (!CreateWindowSurface())
+  if (m_supports_pbuffer && !CreateWindowSurface())
   {
     ERROR_LOG(VIDEO, "Error: CreateWindowSurface failed\n");
+    XSetErrorHandler(oldHandler);
     return false;
   }
+
+  XSetErrorHandler(oldHandler);
   return true;
 }
 
@@ -235,7 +242,7 @@ bool cInterfaceGLX::CreateWindowSurface()
     win = XWindow.CreateXWindow(m_host_window, vi);
     XFree(vi);
   }
-  else
+  else if (m_supports_pbuffer)
   {
     win = m_pbuffer = glXCreateGLXPbufferSGIX(dpy, fbconfig, 1, 1, nullptr);
     if (!m_pbuffer)
@@ -247,11 +254,11 @@ bool cInterfaceGLX::CreateWindowSurface()
 
 void cInterfaceGLX::DestroyWindowSurface()
 {
-  if (!m_pbuffer)
+  if (m_has_handle)
   {
     XWindow.DestroyXWindow();
   }
-  else
+  else if (m_supports_pbuffer && m_pbuffer)
   {
     glXDestroyGLXPbufferSGIX(dpy, m_pbuffer);
     m_pbuffer = 0;

Source/Core/VideoBackends/OGL/ProgramShaderCache.cpp

@@ -1175,6 +1175,24 @@ void ProgramShaderCache::UberShaderCompileWorkItem::Retrieve()
 
 void ProgramShaderCache::CreatePrerenderArrays(SharedContextData* data)
 {
+  // Create a framebuffer object to render into.
+  // This is because in EGL, and potentially GLX, we have a surfaceless context.
+  glGenTextures(1, &data->prerender_FBO_tex);
+  glBindTexture(GL_TEXTURE_2D_ARRAY, data->prerender_FBO_tex);
+  glTexImage3D(GL_TEXTURE_2D_ARRAY, 0, GL_RGBA, 1, 1, 1, 0, GL_RGBA, GL_UNSIGNED_BYTE, nullptr);
+  glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAX_LEVEL, 1);
+  glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  glGenTextures(1, &data->prerender_FBO_depth);
+  glBindTexture(GL_TEXTURE_2D_ARRAY, data->prerender_FBO_depth);
+  glTexImage3D(GL_TEXTURE_2D_ARRAY, 0, GL_DEPTH_COMPONENT32F, 1, 1, 1, 0, GL_DEPTH_COMPONENT,
+               GL_FLOAT, nullptr);
+  glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAX_LEVEL, 1);
+  glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+  glGenFramebuffers(1, &data->prerender_FBO);
+  glBindFramebuffer(GL_FRAMEBUFFER, data->prerender_FBO);
+  glFramebufferTextureLayer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, data->prerender_FBO_tex, 0, 0);
+  glFramebufferTextureLayer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, data->prerender_FBO_depth, 0, 0);
+
   // Create VAO for the prerender vertices.
   // We don't use the normal VAO map, since we need to change the VBO pointer.
   glGenVertexArrays(1, &data->prerender_VAO);
@@ -1257,6 +1275,22 @@ void ProgramShaderCache::DestroyPrerenderArrays(SharedContextData* data)
     glDeleteBuffers(1, &data->prerender_IBO);
     data->prerender_IBO = 0;
   }
+  if (data->prerender_FBO)
+  {
+    glBindFramebuffer(GL_FRAMEBUFFER, 0);
+    glDeleteFramebuffers(1, &data->prerender_FBO);
+    data->prerender_FBO = 0;
+  }
+  if (data->prerender_FBO_tex)
+  {
+    glDeleteTextures(1, &data->prerender_FBO_tex);
+    data->prerender_FBO_tex = 0;
+  }
+  if (data->prerender_FBO_depth)
+  {
+    glDeleteTextures(1, &data->prerender_FBO_depth);
+    data->prerender_FBO_depth = 0;
+  }
 }
 
 void ProgramShaderCache::DrawPrerenderArray(const SHADER& shader, u32 primitive_type)

Source/Core/VideoBackends/OGL/ProgramShaderCache.h

@@ -154,6 +154,9 @@ private:
   struct SharedContextData
   {
     std::unique_ptr<cInterfaceBase> context;
+    GLuint prerender_FBO;
+    GLuint prerender_FBO_tex;
+    GLuint prerender_FBO_depth;
     GLuint prerender_VBO;
     GLuint prerender_VAO;
     GLuint prerender_IBO;