intel: use pwrite for batch

It's faster. Not only is the memcpy more efficiently performed in the
kernel (making up for the system call overhead), but by not using mmap
we remove the greater overhead of tracking the vma of every batch.

And it means we can read back from the batch buffer without incurring
the cost of a uncached read through the GTT.

Signed-off-by: Chris Wilson <chris@chris-wilson.co.uk>

13 files changed, 164 insertions, 235 deletions

diff --git a/src/mesa/drivers/dri/intel/intel_batchbuffer.c b/src/mesa/drivers/dri/intel/intel_batchbuffer.c
index 5564e3ea2c..7d224ae535 100644
--- a/src/mesa/drivers/dri/intel/intel_batchbuffer.c
+++ b/src/mesa/drivers/dri/intel/intel_batchbuffer.c
@@ -34,83 +34,61 @@
 #include "intel_buffers.h"
 
 void
-intel_batchbuffer_reset(struct intel_batchbuffer *batch)
+intel_batchbuffer_reset(struct intel_context *intel)
 {
-   struct intel_context *intel = batch->intel;
-
-   if (batch->buf != NULL) {
-      drm_intel_bo_unreference(batch->buf);
-      batch->buf = NULL;
+   if (intel->batch.bo != NULL) {
+      drm_intel_bo_unreference(intel->batch.bo);
+      intel->batch.bo = NULL;
    }
 
-   batch->buf = drm_intel_bo_alloc(intel->bufmgr, "batchbuffer",
-				   intel->maxBatchSize, 4096);
-   drm_intel_gem_bo_map_gtt(batch->buf);
-   batch->map = batch->buf->virtual;
-
-   batch->size = intel->maxBatchSize;
-   batch->ptr = batch->map;
-   batch->reserved_space = BATCH_RESERVED;
-   batch->dirty_state = ~0;
-   batch->state_batch_offset = batch->size;
-}
-
-struct intel_batchbuffer *
-intel_batchbuffer_alloc(struct intel_context *intel)
-{
-   struct intel_batchbuffer *batch = calloc(sizeof(*batch), 1);
-
-   batch->intel = intel;
-   intel_batchbuffer_reset(batch);
+   intel->batch.bo = drm_intel_bo_alloc(intel->bufmgr, "batchbuffer",
+					intel->maxBatchSize, 4096);
 
-   return batch;
+   intel->batch.reserved_space = BATCH_RESERVED;
+   intel->batch.state_batch_offset = intel->batch.bo->size;
+   intel->batch.used = 0;
 }
 
 void
-intel_batchbuffer_free(struct intel_batchbuffer *batch)
+intel_batchbuffer_free(struct intel_context *intel)
 {
-   if (batch->map) {
-      drm_intel_gem_bo_unmap_gtt(batch->buf);
-      batch->map = NULL;
-   }
-   dri_bo_unreference(batch->buf);
-   batch->buf = NULL;
-   free(batch);
+   drm_intel_bo_unreference(intel->batch.bo);
 }
 
 
-
 /* TODO: Push this whole function into bufmgr.
  */
 static void
-do_flush_locked(struct intel_batchbuffer *batch, GLuint used)
+do_flush_locked(struct intel_context *intel)
 {
-   struct intel_context *intel = batch->intel;
+   struct intel_batchbuffer *batch = &intel->batch;
    int ret = 0;
-   int x_off = 0, y_off = 0;
-
-   drm_intel_gem_bo_unmap_gtt(batch->buf);
-
-   batch->ptr = NULL;
 
    if (!intel->intelScreen->no_hw) {
       int ring;
 
-      if (intel->gen < 6 || !intel->batch->is_blit) {
+      if (intel->gen < 6 || !batch->is_blit) {
 	 ring = I915_EXEC_RENDER;
       } else {
 	 ring = I915_EXEC_BLT;
       }
 
-      drm_intel_bo_mrb_exec(batch->buf, used, NULL, 0,
-			    (x_off & 0xffff) | (y_off << 16), ring);
+      ret = drm_intel_bo_subdata(batch->bo, 0, 4*batch->used, batch->map);
+      if (ret == 0 && batch->state_batch_offset != batch->bo->size) {
+	 ret = drm_intel_bo_subdata(batch->bo,
+				    batch->state_batch_offset,
+				    batch->bo->size - batch->state_batch_offset,
+				    (char *)batch->map + batch->state_batch_offset);
+      }
+
+      if (ret == 0)
+	 ret = drm_intel_bo_mrb_exec(batch->bo, 4*batch->used, NULL, 0, 0, ring);
    }
 
    if (unlikely(INTEL_DEBUG & DEBUG_BATCH)) {
-      drm_intel_bo_map(batch->buf, GL_FALSE);
-      intel_decode(batch->buf->virtual, used / 4, batch->buf->offset,
+      intel_decode(batch->map, batch->used,
+		   batch->bo->offset,
 		   intel->intelScreen->deviceID, GL_TRUE);
-      drm_intel_bo_unmap(batch->buf);
 
       if (intel->vtbl.debug_batch != NULL)
 	 intel->vtbl.debug_batch(intel);
@@ -123,55 +101,33 @@ do_flush_locked(struct intel_batchbuffer *batch, GLuint used)
 }
 
 void
-_intel_batchbuffer_flush(struct intel_batchbuffer *batch, const char *file,
-			 int line)
+_intel_batchbuffer_flush(struct intel_context *intel,
+			 const char *file, int line)
 {
-   struct intel_context *intel = batch->intel;
-   GLuint used = batch->ptr - batch->map;
+   if (intel->batch.used == 0)
+      return;
 
    if (intel->first_post_swapbuffers_batch == NULL) {
-      intel->first_post_swapbuffers_batch = intel->batch->buf;
+      intel->first_post_swapbuffers_batch = intel->batch.bo;
       drm_intel_bo_reference(intel->first_post_swapbuffers_batch);
    }
 
-   if (used == 0)
-      return;
-
    if (unlikely(INTEL_DEBUG & DEBUG_BATCH))
       fprintf(stderr, "%s:%d: Batchbuffer flush with %db used\n", file, line,
-	      used);
+	      4*intel->batch.used);
 
-   batch->reserved_space = 0;
+   intel->batch.reserved_space = 0;
 
    if (intel->always_flush_cache) {
-      intel_batchbuffer_emit_mi_flush(batch);
-      used = batch->ptr - batch->map;
-   }
-
-   /* Round batchbuffer usage to 2 DWORDs. */
-
-   if ((used & 4) == 0) {
-      *(GLuint *) (batch->ptr) = 0; /* noop */
-      batch->ptr += 4;
-      used = batch->ptr - batch->map;
+      intel_batchbuffer_emit_mi_flush(intel);
    }
 
    /* Mark the end of the buffer. */
-   *(GLuint *) (batch->ptr) = MI_BATCH_BUFFER_END;
-   batch->ptr += 4;
-   used = batch->ptr - batch->map;
-   assert (used <= batch->buf->size);
-
-   /* Workaround for recursive batchbuffer flushing: If the window is
-    * moved, we can get into a case where we try to flush during a
-    * flush.  What happens is that when we try to grab the lock for
-    * the first flush, we detect that the window moved which then
-    * causes another flush (from the intel_draw_buffer() call in
-    * intelUpdatePageFlipping()).  To work around this we reset the
-    * batchbuffer tail pointer before trying to get the lock.  This
-    * prevent the nested buffer flush, but a better fix would be to
-    * avoid that in the first place. */
-   batch->ptr = batch->map;
+   intel_batchbuffer_emit_dword(intel, MI_BATCH_BUFFER_END);
+   if (intel->batch.used & 1) {
+      /* Round batchbuffer usage to 2 DWORDs. */
+      intel_batchbuffer_emit_dword(intel, MI_NOOP);
+   }
 
    if (intel->vtbl.finish_batch)
       intel->vtbl.finish_batch(intel);
@@ -181,24 +137,23 @@ _intel_batchbuffer_flush(struct intel_batchbuffer *batch, const char *file,
    /* Check that we didn't just wrap our batchbuffer at a bad time. */
    assert(!intel->no_batch_wrap);
 
-   do_flush_locked(batch, used);
+   do_flush_locked(intel);
 
    if (unlikely(INTEL_DEBUG & DEBUG_SYNC)) {
       fprintf(stderr, "waiting for idle\n");
-      drm_intel_bo_map(batch->buf, GL_TRUE);
-      drm_intel_bo_unmap(batch->buf);
+      drm_intel_bo_wait_rendering(intel->batch.bo);
    }
 
    /* Reset the buffer:
     */
-   intel_batchbuffer_reset(batch);
+   intel_batchbuffer_reset(intel);
 }
 
 
 /*  This is the only way buffers get added to the validate list.
  */
 GLboolean
-intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch,
+intel_batchbuffer_emit_reloc(struct intel_context *intel,
                              drm_intel_bo *buffer,
                              uint32_t read_domains, uint32_t write_domain,
 			     uint32_t delta)
@@ -207,58 +162,55 @@ intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch,
 
    assert(delta < buffer->size);
 
-   if (batch->ptr - batch->map > batch->buf->size)
-    printf ("bad relocation ptr %p map %p offset %d size %lu\n",
-	    batch->ptr, batch->map, batch->ptr - batch->map, batch->buf->size);
-   ret = drm_intel_bo_emit_reloc(batch->buf, batch->ptr - batch->map,
+   ret = drm_intel_bo_emit_reloc(intel->batch.bo, 4*intel->batch.used,
 				 buffer, delta,
 				 read_domains, write_domain);
+   assert (ret == 0);
 
    /*
     * Using the old buffer offset, write in what the right data would be, in case
     * the buffer doesn't move and we can short-circuit the relocation processing
     * in the kernel
     */
-   intel_batchbuffer_emit_dword (batch, buffer->offset + delta);
+   intel_batchbuffer_emit_dword(intel, buffer->offset + delta);
 
    return GL_TRUE;
 }
 
 GLboolean
-intel_batchbuffer_emit_reloc_fenced(struct intel_batchbuffer *batch,
+intel_batchbuffer_emit_reloc_fenced(struct intel_context *intel,
 				    drm_intel_bo *buffer,
-				    uint32_t read_domains, uint32_t write_domain,
+				    uint32_t read_domains,
+				    uint32_t write_domain,
 				    uint32_t delta)
 {
    int ret;
 
    assert(delta < buffer->size);
 
-   if (batch->ptr - batch->map > batch->buf->size)
-    printf ("bad relocation ptr %p map %p offset %d size %lu\n",
-	    batch->ptr, batch->map, batch->ptr - batch->map, batch->buf->size);
-   ret = drm_intel_bo_emit_reloc_fence(batch->buf, batch->ptr - batch->map,
+   ret = drm_intel_bo_emit_reloc_fence(intel->batch.bo, 4*intel->batch.used,
 				       buffer, delta,
 				       read_domains, write_domain);
+   assert (ret == 0);
 
    /*
     * Using the old buffer offset, write in what the right data would
     * be, in case the buffer doesn't move and we can short-circuit the
     * relocation processing in the kernel
     */
-   intel_batchbuffer_emit_dword (batch, buffer->offset + delta);
+   intel_batchbuffer_emit_dword(intel, buffer->offset + delta);
 
    return GL_TRUE;
 }
 
 void
-intel_batchbuffer_data(struct intel_batchbuffer *batch,
+intel_batchbuffer_data(struct intel_context *intel,
                        const void *data, GLuint bytes, bool is_blit)
 {
    assert((bytes & 3) == 0);
-   intel_batchbuffer_require_space(batch, bytes, is_blit);
-   __memcpy(batch->ptr, data, bytes);
-   batch->ptr += bytes;
+   intel_batchbuffer_require_space(intel, bytes, is_blit);
+   __memcpy(intel->batch.map + intel->batch.used, data, bytes);
+   intel->batch.used += bytes >> 2;
 }
 
 /* Emit a pipelined flush to either flush render and texture cache for
@@ -268,12 +220,10 @@ intel_batchbuffer_data(struct intel_batchbuffer *batch,
  * This is also used for the always_flush_cache driconf debug option.
  */
 void
-intel_batchbuffer_emit_mi_flush(struct intel_batchbuffer *batch)
+intel_batchbuffer_emit_mi_flush(struct intel_context *intel)
 {
-   struct intel_context *intel = batch->intel;
-
    if (intel->gen >= 6) {
-      if (intel->batch->is_blit) {
+      if (intel->batch.is_blit) {
 	 BEGIN_BATCH_BLT(4);
 	 OUT_BATCH(MI_FLUSH_DW);
 	 OUT_BATCH(0);
diff --git a/src/mesa/drivers/dri/intel/intel_batchbuffer.h b/src/mesa/drivers/dri/intel/intel_batchbuffer.h
index 635708587a..7699715c0e 100644
--- a/src/mesa/drivers/dri/intel/intel_batchbuffer.h
+++ b/src/mesa/drivers/dri/intel/intel_batchbuffer.h
@@ -7,71 +7,37 @@
 #include "intel_bufmgr.h"
 #include "intel_reg.h"
 
-#define BATCH_SZ 16384
 #define BATCH_RESERVED 16
 
+void intel_batchbuffer_reset(struct intel_context *intel);
+void intel_batchbuffer_free(struct intel_context *intel);
 
-struct intel_batchbuffer
-{
-   struct intel_context *intel;
-
-   drm_intel_bo *buf;
-
-   GLubyte *map;
-   GLubyte *ptr;
-
-   GLuint size;
-   uint32_t state_batch_offset;
-
-#ifdef DEBUG
-   /** Tracking of BEGIN_BATCH()/OUT_BATCH()/ADVANCE_BATCH() debugging */
-   struct {
-      GLuint total;
-      GLubyte *start_ptr;
-   } emit;
-#endif
-
-   bool is_blit;
-   GLuint dirty_state;
-   GLuint reserved_space;
-};
-
-struct intel_batchbuffer *intel_batchbuffer_alloc(struct intel_context
-                                                  *intel);
-
-void intel_batchbuffer_free(struct intel_batchbuffer *batch);
-
-
-void _intel_batchbuffer_flush(struct intel_batchbuffer *batch,
+void _intel_batchbuffer_flush(struct intel_context *intel,
 			      const char *file, int line);
 
-#define intel_batchbuffer_flush(batch) \
-	_intel_batchbuffer_flush(batch, __FILE__, __LINE__)
+#define intel_batchbuffer_flush(intel) \
+	_intel_batchbuffer_flush(intel, __FILE__, __LINE__)
 
-void intel_batchbuffer_reset(struct intel_batchbuffer *batch);
 
 
 /* Unlike bmBufferData, this currently requires the buffer be mapped.
  * Consider it a convenience function wrapping multple
  * intel_buffer_dword() calls.
  */
-void intel_batchbuffer_data(struct intel_batchbuffer *batch,
+void intel_batchbuffer_data(struct intel_context *intel,
                             const void *data, GLuint bytes, bool is_blit);
 
-void intel_batchbuffer_release_space(struct intel_batchbuffer *batch,
-                                     GLuint bytes);
-
-GLboolean intel_batchbuffer_emit_reloc(struct intel_batchbuffer *batch,
+GLboolean intel_batchbuffer_emit_reloc(struct intel_context *intel,
                                        drm_intel_bo *buffer,
 				       uint32_t read_domains,
 				       uint32_t write_domain,
 				       uint32_t offset);
-GLboolean intel_batchbuffer_emit_reloc_fenced(struct intel_batchbuffer *batch,
+GLboolean intel_batchbuffer_emit_reloc_fenced(struct intel_context *intel,
 					      drm_intel_bo *buffer,
 					      uint32_t read_domains,
 					      uint32_t write_domain,
 					      uint32_t offset);
-void intel_batchbuffer_emit_mi_flush(struct intel_batchbuffer *batch);
+void intel_batchbuffer_emit_mi_flush(struct intel_context *intel);
 
 static INLINE uint32_t float_as_int(float f)
 {
@@ -90,73 +56,70 @@ static INLINE uint32_t float_as_int(float f)
  * work...
  */
 static INLINE GLint
-intel_batchbuffer_space(struct intel_batchbuffer *batch)
+intel_batchbuffer_space(struct intel_context *intel)
 {
-   return (batch->state_batch_offset - batch->reserved_space) -
-      (batch->ptr - batch->map);
+   return (intel->batch.state_batch_offset - intel->batch.reserved_space) - intel->batch.used*4;
 }
 
 
 static INLINE void
-intel_batchbuffer_emit_dword(struct intel_batchbuffer *batch, GLuint dword)
+intel_batchbuffer_emit_dword(struct intel_context *intel, GLuint dword)
 {
 #ifdef DEBUG
-   assert(intel_batchbuffer_space(batch) >= 4);
+   assert(intel_batchbuffer_space(intel) >= 4);
 #endif
-   *(GLuint *) (batch->ptr) = dword;
-   batch->ptr += 4;
+   intel->batch.map[intel->batch.used++] = dword;
 }
 
 static INLINE void
-intel_batchbuffer_emit_float(struct intel_batchbuffer *batch, float f)
+intel_batchbuffer_emit_float(struct intel_context *intel, float f)
 {
-   intel_batchbuffer_emit_dword(batch, float_as_int(f));
+   intel_batchbuffer_emit_dword(intel, float_as_int(f));
 }
 
 static INLINE void
-intel_batchbuffer_require_space(struct intel_batchbuffer *batch,
+intel_batchbuffer_require_space(struct intel_context *intel,
                                 GLuint sz, int is_blit)
 {
 
-   if (batch->intel->gen >= 6 && batch->is_blit != is_blit &&
-       batch->ptr != batch->map) {
-      intel_batchbuffer_flush(batch);
+   if (intel->gen >= 6 &&
+       intel->batch.is_blit != is_blit && intel->batch.used) {
+      intel_batchbuffer_flush(intel);
    }
 
-   batch->is_blit = is_blit;
+   intel->batch.is_blit = is_blit;
 
 #ifdef DEBUG
-   assert(sz < batch->size - 8);
+   assert(sz < sizeof(intel->batch.map) - BATCH_RESERVED);
 #endif
-   if (intel_batchbuffer_space(batch) < sz)
-      intel_batchbuffer_flush(batch);
+   if (intel_batchbuffer_space(intel) < sz)
+      intel_batchbuffer_flush(intel);
 }
 
 static INLINE void
-intel_batchbuffer_begin(struct intel_batchbuffer *batch, int n, bool is_blit)
+intel_batchbuffer_begin(struct intel_context *intel, int n, bool is_blit)
 {
-   intel_batchbuffer_require_space(batch, n * 4, is_blit);
+   intel_batchbuffer_require_space(intel, n * 4, is_blit);
 
 #ifdef DEBUG
-   assert(batch->map);
-   assert(batch->emit.start_ptr == NULL);
-   batch->emit.total = n * 4;
-   batch->emit.start_ptr = batch->ptr;
+   intel->batch.emit.total = n;
+   intel->batch.emit.start_ptr = intel->batch.used;
 #endif
 }
 
 static INLINE void
-intel_batchbuffer_advance(struct intel_batchbuffer *batch)
+intel_batchbuffer_advance(struct intel_context *intel)
 {
 #ifdef DEBUG
-   unsigned int _n = batch->ptr - batch->emit.start_ptr;
-   assert(batch->emit.start_ptr != NULL);
-   if (_n != batch->emit.total) {
+   struct intel_batchbuffer *batch = &intel->batch;
+   unsigned int _n = batch->used - batch->emit;
+   assert(batch->total != 0);
+   if (_n != batch->total) {
       fprintf(stderr, "ADVANCE_BATCH: %d of %d dwords emitted\n",
-	      _n, batch->emit.total);
+	      _n, batch->total);
       abort();
    }
-   batch->emit.start_ptr = NULL;
+   batch->total = 0;
 #endif
 }
 
@@ -164,19 +127,19 @@ intel_batchbuffer_advance(struct intel_batchbuffer *batch)
  */
 #define BATCH_LOCALS
 
-#define BEGIN_BATCH(n) intel_batchbuffer_begin(intel->batch, n, false)
-#define BEGIN_BATCH_BLT(n) intel_batchbuffer_begin(intel->batch, n, true)
-#define OUT_BATCH(d) intel_batchbuffer_emit_dword(intel->batch, d)
-#define OUT_BATCH_F(f) intel_batchbuffer_emit_float(intel->batch,f)
+#define BEGIN_BATCH(n) intel_batchbuffer_begin(intel, n, false)
+#define BEGIN_BATCH_BLT(n) intel_batchbuffer_begin(intel, n, true)
+#define OUT_BATCH(d) intel_batchbuffer_emit_dword(intel, d)
+#define OUT_BATCH_F(f) intel_batchbuffer_emit_float(intel,f)
 #define OUT_RELOC(buf, read_domains, write_domain, delta) do {		\
-   intel_batchbuffer_emit_reloc(intel->batch, buf,			\
+   intel_batchbuffer_emit_reloc(intel, buf,			\
 				read_domains, write_domain, delta);	\
 } while (0)
 #define OUT_RELOC_FENCED(buf, read_domains, write_domain, delta) do {	\
-   intel_batchbuffer_emit_reloc_fenced(intel->batch, buf,		\
+   intel_batchbuffer_emit_reloc_fenced(intel, buf,		\
 				       read_domains, write_domain, delta); \
 } while (0)
 
-#define ADVANCE_BATCH() intel_batchbuffer_advance(intel->batch);
+#define ADVANCE_BATCH() intel_batchbuffer_advance(intel);
 
 #endif
diff --git a/src/mesa/drivers/dri/intel/intel_blit.c b/src/mesa/drivers/dri/intel/intel_blit.c
index 6232e479cb..e1ab7f1637 100644
--- a/src/mesa/drivers/dri/intel/intel_blit.c
+++ b/src/mesa/drivers/dri/intel/intel_blit.c
@@ -123,12 +123,12 @@ intelEmitCopyBlit(struct intel_context *intel,
 
    /* do space check before going any further */
    do {
-       aper_array[0] = intel->batch->buf;
+       aper_array[0] = intel->batch.bo;
        aper_array[1] = dst_buffer;
        aper_array[2] = src_buffer;
 
        if (dri_bufmgr_check_aperture_space(aper_array, 3) != 0) {
-           intel_batchbuffer_flush(intel->batch);
+           intel_batchbuffer_flush(intel);
            pass++;
        } else
            break;
@@ -137,7 +137,7 @@ intelEmitCopyBlit(struct intel_context *intel,
    if (pass >= 2)
       return GL_FALSE;
 
-   intel_batchbuffer_require_space(intel->batch, 8 * 4, true);
+   intel_batchbuffer_require_space(intel, 8 * 4, true);
    DBG("%s src:buf(%p)/%d+%d %d,%d dst:buf(%p)/%d+%d %d,%d sz:%dx%d\n",
        __FUNCTION__,
        src_buffer, src_pitch, src_offset, src_x, src_y,
@@ -193,7 +193,7 @@ intelEmitCopyBlit(struct intel_context *intel,
 		    src_offset);
    ADVANCE_BATCH();
 
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 
    return GL_TRUE;
 }
@@ -343,12 +343,12 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
       assert(y1 < y2);
 
       /* do space check before going any further */
-      aper_array[0] = intel->batch->buf;
+      aper_array[0] = intel->batch.bo;
       aper_array[1] = write_buffer;
 
       if (drm_intel_bufmgr_check_aperture_space(aper_array,
 						ARRAY_SIZE(aper_array)) != 0) {
-	 intel_batchbuffer_flush(intel->batch);
+	 intel_batchbuffer_flush(intel);
       }
 
       BEGIN_BATCH_BLT(6);
@@ -363,7 +363,7 @@ intelClearWithBlit(struct gl_context *ctx, GLbitfield mask)
       ADVANCE_BATCH();
 
       if (intel->always_flush_cache)
-	 intel_batchbuffer_emit_mi_flush(intel->batch);
+	 intel_batchbuffer_emit_mi_flush(intel);
 
       if (buf == BUFFER_DEPTH || buf == BUFFER_STENCIL)
 	 mask &= ~(BUFFER_BIT_DEPTH | BUFFER_BIT_STENCIL);
@@ -410,10 +410,10 @@ intelEmitImmediateColorExpandBlit(struct intel_context *intel,
        __FUNCTION__,
        dst_buffer, dst_pitch, dst_offset, x, y, w, h, src_size, dwords);
 
-   intel_batchbuffer_require_space( intel->batch,
-				    (8 * 4) +
-				    (3 * 4) +
-				    dwords * 4, true);
+   intel_batchbuffer_require_space(intel,
+				   (8 * 4) +
+				   (3 * 4) +
+				   dwords * 4, true);
 
    opcode = XY_SETUP_BLT_CMD;
    if (cpp == 4)
@@ -449,11 +449,9 @@ intelEmitImmediateColorExpandBlit(struct intel_context *intel,
    OUT_BATCH(((y + h) << 16) | (x + w));
    ADVANCE_BATCH();
 
-   intel_batchbuffer_data(intel->batch,
-			  src_bits,
-			  dwords * 4, true);
+   intel_batchbuffer_data(intel, src_bits, dwords * 4, true);
 
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 
    return GL_TRUE;
 }
@@ -543,10 +541,10 @@ intel_set_teximage_alpha_to_one(struct gl_context *ctx,
 
    DBG("%s dst:buf(%p)/%d %d,%d sz:%dx%d\n",
        __FUNCTION__,
-       intel_image->mt->region->buffer, (pitch * region->cpp),
+       intel_image->mt->region->buffer, (pitch * cpp),
        x1, y1, x2 - x1, y2 - y1);
 
-   BR13 = br13_for_cpp(region->cpp) | 0xf0 << 16;
+   BR13 = br13_for_cpp(cpp) | 0xf0 << 16;
    CMD = XY_COLOR_BLT_CMD;
    CMD |= XY_BLT_WRITE_ALPHA;
 
@@ -558,15 +556,15 @@ intel_set_teximage_alpha_to_one(struct gl_context *ctx,
       pitch /= 4;
    }
 #endif
-   BR13 |= (pitch * region->cpp);
+   BR13 |= (pitch * cpp);
 
    /* do space check before going any further */
-   aper_array[0] = intel->batch->buf;
+   aper_array[0] = intel->batch.bo;
    aper_array[1] = region->buffer;
 
    if (drm_intel_bufmgr_check_aperture_space(aper_array,
 					     ARRAY_SIZE(aper_array)) != 0) {
-      intel_batchbuffer_flush(intel->batch);
+      intel_batchbuffer_flush(intel);
    }
 
    BEGIN_BATCH_BLT(6);
@@ -580,5 +578,5 @@ intel_set_teximage_alpha_to_one(struct gl_context *ctx,
    OUT_BATCH(0xffffffff); /* white, but only alpha gets written */
    ADVANCE_BATCH();
 
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 }
diff --git a/src/mesa/drivers/dri/intel/intel_buffer_objects.c b/src/mesa/drivers/dri/intel/intel_buffer_objects.c
index 19c73e6ab3..4edf29ec75 100644
--- a/src/mesa/drivers/dri/intel/intel_buffer_objects.c
+++ b/src/mesa/drivers/dri/intel/intel_buffer_objects.c
@@ -225,7 +225,7 @@ intel_bufferobj_subdata(struct gl_context * ctx,
    } else {
       bool busy =
 	 drm_intel_bo_busy(intel_obj->buffer) ||
-	 drm_intel_bo_references(intel->batch->buf, intel_obj->buffer);
+	 drm_intel_bo_references(intel->batch.bo, intel_obj->buffer);
 
       /* replace the current busy bo with fresh data */
       if (busy && size == intel_obj->Base.Size) {
@@ -251,8 +251,8 @@ intel_bufferobj_subdata(struct gl_context * ctx,
 	 }
       } else {
 	 /* Can't use the blit to modify the buffer in the middle of batch. */
-	 if (drm_intel_bo_references(intel->batch->buf, intel_obj->buffer)) {
-	    intel_batchbuffer_flush(intel->batch);
+	 if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer)) {
+	    intel_batchbuffer_flush(intel);
 	 }
 	 drm_intel_bo_subdata(intel_obj->buffer, offset, size, data);
       }
@@ -309,7 +309,7 @@ intel_bufferobj_map(struct gl_context * ctx,
    }
 
    /* Flush any existing batchbuffer that might reference this data. */
-   if (drm_intel_bo_references(intel->batch->buf, intel_obj->buffer))
+   if (drm_intel_bo_references(intel->batch.bo, intel_obj->buffer))
       intel_flush(ctx);
 
    if (intel_obj->region)
@@ -386,7 +386,7 @@ intel_bufferobj_map_range(struct gl_context * ctx,
     * syncing.
     */
    if (!(access & GL_MAP_UNSYNCHRONIZED_BIT) &&
-       drm_intel_bo_references(intel->batch->buf, intel_obj->buffer))
+       drm_intel_bo_references(intel->batch.bo, intel_obj->buffer))
       intel_flush(ctx);
 
    if (intel_obj->buffer == NULL) {
@@ -499,7 +499,7 @@ intel_bufferobj_unmap(struct gl_context * ctx,
        * flush.  Once again, we wish for a domain tracker in libdrm to cover
        * usage inside of a batchbuffer.
        */
-      intel_batchbuffer_emit_mi_flush(intel->batch);
+      intel_batchbuffer_emit_mi_flush(intel);
       free(intel_obj->range_map_buffer);
       intel_obj->range_map_buffer = NULL;
    } else if (intel_obj->range_map_bo != NULL) {
@@ -519,7 +519,7 @@ intel_bufferobj_unmap(struct gl_context * ctx,
        * flush.  Once again, we wish for a domain tracker in libdrm to cover
        * usage inside of a batchbuffer.
        */
-      intel_batchbuffer_emit_mi_flush(intel->batch);
+      intel_batchbuffer_emit_mi_flush(intel);
 
       drm_intel_bo_unreference(intel_obj->range_map_bo);
       intel_obj->range_map_bo = NULL;
@@ -766,7 +766,7 @@ intel_bufferobj_copy_subdata(struct gl_context *ctx,
     * flush.  Once again, we wish for a domain tracker in libdrm to cover
     * usage inside of a batchbuffer.
     */
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 }
 
 #if FEATURE_APPLE_object_purgeable
diff --git a/src/mesa/drivers/dri/intel/intel_context.c b/src/mesa/drivers/dri/intel/intel_context.c
index 65c4148e73..c470febfb3 100644
--- a/src/mesa/drivers/dri/intel/intel_context.c
+++ b/src/mesa/drivers/dri/intel/intel_context.c
@@ -554,8 +554,8 @@ intel_flush(struct gl_context *ctx)
    if (intel->gen < 4)
       INTEL_FIREVERTICES(intel);
 
-   if (intel->batch->map != intel->batch->ptr)
-      intel_batchbuffer_flush(intel->batch);
+   if (intel->batch.used)
+      intel_batchbuffer_flush(intel);
 }
 
 static void
@@ -751,7 +751,7 @@ intelInitContext(struct intel_context *intel,
    if (intelScreen->deviceID == PCI_CHIP_I865_G)
       intel->maxBatchSize = 4096;
    else
-      intel->maxBatchSize = BATCH_SZ;
+      intel->maxBatchSize = sizeof(intel->batch.map);
 
    intel->bufmgr = intelScreen->bufmgr;
 
@@ -863,7 +863,7 @@ intelInitContext(struct intel_context *intel,
    if (INTEL_DEBUG & DEBUG_BUFMGR)
       dri_bufmgr_set_debug(intel->bufmgr, GL_TRUE);
 
-   intel->batch = intel_batchbuffer_alloc(intel);
+   intel_batchbuffer_reset(intel);
 
    intel_fbo_init(intel);
 
@@ -920,8 +920,7 @@ intelDestroyContext(__DRIcontext * driContextPriv)
       _swrast_DestroyContext(&intel->ctx);
       intel->Fallback = 0x0;      /* don't call _swrast_Flush later */
 
-      intel_batchbuffer_free(intel->batch);
-      intel->batch = NULL;
+      intel_batchbuffer_free(intel);
 
       free(intel->prim.vb);
       intel->prim.vb = NULL;
diff --git a/src/mesa/drivers/dri/intel/intel_context.h b/src/mesa/drivers/dri/intel/intel_context.h
index 0ea273837d..bf2a0b4ead 100644
--- a/src/mesa/drivers/dri/intel/intel_context.h
+++ b/src/mesa/drivers/dri/intel/intel_context.h
@@ -169,7 +169,27 @@ struct intel_context
 
    int urb_size;
 
-   struct intel_batchbuffer *batch;
+   struct intel_batchbuffer {
+      drm_intel_bo *bo;
+
+      uint16_t used;
+      uint16_t reserved_space;
+      uint32_t map[8192];
+#define BATCH_SZ (8192*sizeof(uint32_t))
+
+      uint32_t state_batch_offset;
+
+#ifdef DEBUG
+      /** Tracking of BEGIN_BATCH()/OUT_BATCH()/ADVANCE_BATCH() debugging */
+      struct {
+	 uint16_t total;
+	 uint16_t start_ptr;
+      } emit;
+#endif
+
+      bool is_blit;
+   } batch;
+
    drm_intel_bo *first_post_swapbuffers_batch;
    GLboolean need_throttle;
    GLboolean no_batch_wrap;
@@ -177,9 +197,9 @@ struct intel_context
    struct
    {
       GLuint id;
+      uint32_t start_ptr; /**< for i8xx */
       uint32_t primitive;	/**< Current hardware primitive type */
       void (*flush) (struct intel_context *);
-      GLubyte *start_ptr; /**< for i8xx */
       drm_intel_bo *vb_bo;
       uint8_t *vb;
       unsigned int start_offset; /**< Byte offset of primitive sequence */
diff --git a/src/mesa/drivers/dri/intel/intel_fbo.c b/src/mesa/drivers/dri/intel/intel_fbo.c
index 0db5a491c8..886b25c23b 100644
--- a/src/mesa/drivers/dri/intel/intel_fbo.c
+++ b/src/mesa/drivers/dri/intel/intel_fbo.c
@@ -580,7 +580,7 @@ intel_finish_render_texture(struct gl_context * ctx,
     * batch.  Once again, we wish for a domain tracker in libdrm to cover
     * usage inside of a batchbuffer like GEM does in the kernel.
     */
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 }
 
 /**
diff --git a/src/mesa/drivers/dri/intel/intel_pixel_bitmap.c b/src/mesa/drivers/dri/intel/intel_pixel_bitmap.c
index e7356a6da0..d7561ee689 100644
--- a/src/mesa/drivers/dri/intel/intel_pixel_bitmap.c
+++ b/src/mesa/drivers/dri/intel/intel_pixel_bitmap.c
@@ -285,7 +285,7 @@ do_blit_bitmap( struct gl_context *ctx,
 out:
 
    if (unlikely(INTEL_DEBUG & DEBUG_SYNC))
-      intel_batchbuffer_flush(intel->batch);
+      intel_batchbuffer_flush(intel);
 
    if (_mesa_is_bufferobj(unpack->BufferObj)) {
       /* done with PBO so unmap it now */
diff --git a/src/mesa/drivers/dri/intel/intel_pixel_read.c b/src/mesa/drivers/dri/intel/intel_pixel_read.c
index 716b9cea40..f90f8224c9 100644
--- a/src/mesa/drivers/dri/intel/intel_pixel_read.c
+++ b/src/mesa/drivers/dri/intel/intel_pixel_read.c
@@ -78,7 +78,6 @@ do_blit_readpixels(struct gl_context * ctx,
    GLuint dst_offset;
    GLuint rowLength;
    drm_intel_bo *dst_buffer;
-   GLuint offset;
    GLboolean all;
    GLint dst_x, dst_y;
 
@@ -148,7 +147,7 @@ do_blit_readpixels(struct gl_context * ctx,
    if (!intelEmitCopyBlit(intel,
 			  src->cpp,
 			  src->pitch, src->buffer, 0, src->tiling,
-			  rowLength, dst_buffer, dst_offset + offset, GL_FALSE,
+			  rowLength, dst_buffer, dst_offset, GL_FALSE,
 			  x, y,
 			  dst_x, dst_y,
 			  width, height,
diff --git a/src/mesa/drivers/dri/intel/intel_regions.c b/src/mesa/drivers/dri/intel/intel_regions.c
index e87e29462c..0857fa8ad7 100644
--- a/src/mesa/drivers/dri/intel/intel_regions.c
+++ b/src/mesa/drivers/dri/intel/intel_regions.c
@@ -491,7 +491,7 @@ intel_region_cow(struct intel_context *intel, struct intel_region *region)
 
    assert(region->cpp * region->pitch * region->height == pbo->Base.Size);
 
-   _DBG("%s %p (%d bytes)\n", __FUNCTION__, region, pbo->Base.Size);
+   _DBG("%s %p (%d bytes)\n", __FUNCTION__, region, (int)pbo->Base.Size);
 
    /* Now blit from the texture buffer to the new buffer: 
     */
diff --git a/src/mesa/drivers/dri/intel/intel_screen.c b/src/mesa/drivers/dri/intel/intel_screen.c
index 2f35fe7f24..356d5f72d8 100644
--- a/src/mesa/drivers/dri/intel/intel_screen.c
+++ b/src/mesa/drivers/dri/intel/intel_screen.c
@@ -111,8 +111,8 @@ intelDRI2Flush(__DRIdrawable *drawable)
 
    intel->need_throttle = GL_TRUE;
 
-   if (intel->batch->map != intel->batch->ptr)
-      intel_batchbuffer_flush(intel->batch);
+   if (intel->batch.used)
+      intel_batchbuffer_flush(intel);
 }
 
 static const struct __DRI2flushExtensionRec intelFlushExtension = {
diff --git a/src/mesa/drivers/dri/intel/intel_syncobj.c b/src/mesa/drivers/dri/intel/intel_syncobj.c
index bbfac74b60..b303ea84dd 100644
--- a/src/mesa/drivers/dri/intel/intel_syncobj.c
+++ b/src/mesa/drivers/dri/intel/intel_syncobj.c
@@ -72,9 +72,9 @@ intel_fence_sync(struct gl_context *ctx, struct gl_sync_object *s,
    struct intel_sync_object *sync = (struct intel_sync_object *)s;
 
    assert(condition == GL_SYNC_GPU_COMMANDS_COMPLETE);
-   intel_batchbuffer_emit_mi_flush(intel->batch);
+   intel_batchbuffer_emit_mi_flush(intel);
 
-   sync->bo = intel->batch->buf;
+   sync->bo = intel->batch.bo;
    drm_intel_bo_reference(sync->bo);
 
    intel_flush(ctx);
diff --git a/src/mesa/drivers/dri/intel/intel_tex_image.c b/src/mesa/drivers/dri/intel/intel_tex_image.c
index ddcb748c82..9dba529c58 100644
--- a/src/mesa/drivers/dri/intel/intel_tex_image.c
+++ b/src/mesa/drivers/dri/intel/intel_tex_image.c
@@ -231,7 +231,7 @@ try_pbo_upload(struct intel_context *intel,
 
    dst_stride = intelImage->mt->region->pitch;
 
-   if (drm_intel_bo_references(intel->batch->buf, dst_buffer))
+   if (drm_intel_bo_references(intel->batch.bo, dst_buffer))
       intel_flush(&intel->ctx);
 
    {
@@ -431,7 +431,7 @@ intelTexImage(struct gl_context * ctx,
    if (intelImage->mt) {
       if (pixels != NULL) {
 	 /* Flush any queued rendering with the texture before mapping. */
-	 if (drm_intel_bo_references(intel->batch->buf,
+	 if (drm_intel_bo_references(intel->batch.bo,
 				     intelImage->mt->region->buffer)) {
 	    intel_flush(ctx);
 	 }