Assorted fixes for dealing with zero-size frame/renderbuffers.

In xmesa_check_and_update_buffer_size() handle xmctx==NULL correctly: still
call _mesa_resize_framebufer().  If we don't we can wind up in a situation
where the framebuffer size is non-zero but an attached renderbuffer size
is still initialized to zero.  This inconsistancy can later cause problems.
Check for zero-size renderbuffers in update_color_draw_buffers() and
update_color_read_buffer().
See bug 7205.

4 files changed, 51 insertions, 35 deletions

diff --git a/src/mesa/drivers/x11/xm_api.c b/src/mesa/drivers/x11/xm_api.c
index ba020fc3d6..b513dc8d40 100644
--- a/src/mesa/drivers/x11/xm_api.c
+++ b/src/mesa/drivers/x11/xm_api.c
@@ -1842,16 +1842,18 @@ XMesaDestroyBuffer(XMesaBuffer b)
  *  1. the first time a buffer is bound to a context.
  *  2. from glViewport to poll for window size changes
  *  3. from the XMesaResizeBuffers() API function.
+ * Note: it's possible (and legal) for xmctx to be NULL.  That can happen
+ * when resizing a buffer when no rendering context is bound.
  */
 void
 xmesa_check_and_update_buffer_size(XMesaContext xmctx, XMesaBuffer drawBuffer)
 {
    GLuint width, height;
-   xmesa_get_window_size(xmctx->display, drawBuffer, &width, &height);
+   xmesa_get_window_size(drawBuffer->display, drawBuffer, &width, &height);
    if (drawBuffer->mesa_buffer.Width != width ||
        drawBuffer->mesa_buffer.Height != height) {
-      _mesa_resize_framebuffer(&(xmctx->mesa),
-                               &(drawBuffer->mesa_buffer), width, height);
+      GLcontext *ctx = xmctx ? &xmctx->mesa : NULL;
+      _mesa_resize_framebuffer(ctx, &(drawBuffer->mesa_buffer), width, height);
    }
    drawBuffer->mesa_buffer.Initialized = GL_TRUE; /* XXX TEMPORARY? */
 }
@@ -2175,7 +2177,7 @@ void XMesaSwapBuffers( XMesaBuffer b )
       }
 #endif
      if (b->backxrb->ximage) {
-	 /* Copy Ximage from host's memory to server's window */
+	 /* Copy Ximage (back buf) from client memory to server window */
 #if defined(USE_XSHM) && !defined(XFree86Server)
 	 if (b->shm) {
             /*_glthread_LOCK_MUTEX(_xmesa_lock);*/
@@ -2197,8 +2199,8 @@ void XMesaSwapBuffers( XMesaBuffer b )
             /*_glthread_UNLOCK_MUTEX(_xmesa_lock);*/
          }
       }
-      else {
-	 /* Copy pixmap to window on server */
+      else if (b->backxrb->pixmap) {
+	 /* Copy pixmap (back buf) to window (front buf) on server */
          /*_glthread_LOCK_MUTEX(_xmesa_lock);*/
 	 XMesaCopyArea( b->xm_visual->display,
 			b->backxrb->pixmap,   /* source drawable */
diff --git a/src/mesa/drivers/x11/xm_buffer.c b/src/mesa/drivers/x11/xm_buffer.c
index 73c46b1fe6..c1fa23328f 100644
--- a/src/mesa/drivers/x11/xm_buffer.c
+++ b/src/mesa/drivers/x11/xm_buffer.c
@@ -168,9 +168,6 @@ alloc_back_shm_ximage(XMesaBuffer b, GLuint width, GLuint height)
 static void
 alloc_back_buffer(XMesaBuffer b, GLuint width, GLuint height)
 {
-   if (width == 0 || height == 0)
-      return;
-
    if (b->db_mode == BACK_XIMAGE) {
       /* Deallocate the old backxrb->ximage, if any */
       if (b->backxrb->ximage) {
@@ -186,6 +183,9 @@ alloc_back_buffer(XMesaBuffer b, GLuint width, GLuint height)
 	 b->backxrb->ximage = NULL;
       }
 
+      if (width == 0 || height == 0)
+         return;
+
       /* Allocate new back buffer */
 #ifdef XFree86Server
       /* Allocate a regular XImage for the back buffer. */
@@ -218,20 +218,20 @@ alloc_back_buffer(XMesaBuffer b, GLuint width, GLuint height)
       b->backxrb->pixmap = None;
    }
    else if (b->db_mode == BACK_PIXMAP) {
-      if (!width)
-         width = 1;
-      if (!height)
-         height = 1;
-
       /* Free the old back pixmap */
       if (b->backxrb->pixmap) {
-	 XMesaFreePixmap(b->xm_visual->display, b->backxrb->pixmap);
+         XMesaFreePixmap(b->xm_visual->display, b->backxrb->pixmap);
+         b->backxrb->pixmap = 0;
       }
-      /* Allocate new back pixmap */
-      b->backxrb->pixmap = XMesaCreatePixmap(b->xm_visual->display,
-                                             b->frontxrb->drawable,
-                                             width, height,
-                                             GET_VISUAL_DEPTH(b->xm_visual));
+
+      if (width > 0 && height > 0) {
+         /* Allocate new back pixmap */
+         b->backxrb->pixmap = XMesaCreatePixmap(b->xm_visual->display,
+                                                b->frontxrb->drawable,
+                                                width, height,
+                                                GET_VISUAL_DEPTH(b->xm_visual));
+      }
+
       b->backxrb->ximage = NULL;
    }
 }
@@ -250,6 +250,7 @@ xmesa_delete_renderbuffer(struct gl_renderbuffer *rb)
 
 /**
  * Reallocate renderbuffer storage for front color buffer.
+ * Called via gl_renderbuffer::AllocStorage()
  */
 static GLboolean
 xmesa_alloc_front_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
@@ -260,6 +261,7 @@ xmesa_alloc_front_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
    /* just clear these to be sure we don't accidentally use them */
    xrb->origin1 = NULL;
    xrb->origin2 = NULL;
+   xrb->origin3 = NULL;
    xrb->origin4 = NULL;
 
    /* for the FLIP macro: */
@@ -275,6 +277,7 @@ xmesa_alloc_front_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
 
 /**
  * Reallocate renderbuffer storage for back color buffer.
+ * Called via gl_renderbuffer::AllocStorage()
  */
 static GLboolean
 xmesa_alloc_back_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
@@ -309,8 +312,12 @@ xmesa_alloc_back_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
       xrb->origin4 = (GLuint *) xrb->ximage->data + xrb->width4 * (height - 1);
    }
    else {
-      /* this assertion will fail if we happend to run out of memory */
-      /*assert(xrb->pixmap);*/
+      /* out of memory or buffer size is 0 x 0 */
+      xrb->width1 = xrb->width2 = xrb->width3 = xrb->width4 = 0;
+      xrb->origin1 = NULL;
+      xrb->origin2 = NULL;
+      xrb->origin3 = NULL;
+      xrb->origin4 = NULL;
    }
 
    return GL_TRUE;
diff --git a/src/mesa/main/framebuffer.c b/src/mesa/main/framebuffer.c
index 3136a950e0..cd4f594aa2 100644
--- a/src/mesa/main/framebuffer.c
+++ b/src/mesa/main/framebuffer.c
@@ -647,7 +647,7 @@ update_color_draw_buffers(GLcontext *ctx, struct gl_framebuffer *fb)
             const GLuint bufferBit = 1 << i;
             if (bufferBit & bufferMask) {
                struct gl_renderbuffer *rb = fb->Attachment[i].Renderbuffer;
-               if (rb) {
+               if (rb && rb->Width > 0 && rb->Height > 0) {
                   fb->_ColorDrawBuffers[output][count] = rb;
                   count++;
                }
@@ -673,7 +673,10 @@ static void
 update_color_read_buffer(GLcontext *ctx, struct gl_framebuffer *fb)
 {
    (void) ctx;
-   if (fb->_ColorReadBufferIndex == -1 || fb->DeletePending) {
+   if (fb->_ColorReadBufferIndex == -1 ||
+       fb->DeletePending ||
+       fb->Width == 0 ||
+       fb->Height == 0) {
       fb->_ColorReadBuffer = NULL; /* legal! */
    }
    else {
diff --git a/src/mesa/main/renderbuffer.c b/src/mesa/main/renderbuffer.c
index 1cc95a7d3b..e387c42c34 100644
--- a/src/mesa/main/renderbuffer.c
+++ b/src/mesa/main/renderbuffer.c
@@ -1192,18 +1192,22 @@ _mesa_soft_renderbuffer_storage(GLcontext *ctx, struct gl_renderbuffer *rb,
    ASSERT(rb->PutMonoValues);
 
    /* free old buffer storage */
-   if (rb->Data)
+   if (rb->Data) {
       _mesa_free(rb->Data);
-
-   /* allocate new buffer storage */
-   rb->Data = _mesa_malloc(width * height * pixelSize);
-   if (rb->Data == NULL) {
-      rb->Width = 0;
-      rb->Height = 0;
-      _mesa_error(ctx, GL_OUT_OF_MEMORY,
-                  "software renderbuffer allocation (%d x %d x %d)",
-                  width, height, pixelSize);
-      return GL_FALSE;
+      rb->Data = NULL;
+   }
+
+   if (width > 0 && height > 0) {
+      /* allocate new buffer storage */
+      rb->Data = _mesa_malloc(width * height * pixelSize);
+      if (rb->Data == NULL) {
+         rb->Width = 0;
+         rb->Height = 0;
+         _mesa_error(ctx, GL_OUT_OF_MEMORY,
+                     "software renderbuffer allocation (%d x %d x %d)",
+                     width, height, pixelSize);
+         return GL_FALSE;
+      }
    }
 
    rb->Width = width;