r600g: add upload manager support.

this add support for the upload manager for uploading user vbo/index buffers.

this provides a considerable speedup in q3 type games.

6 files changed, 222 insertions, 46 deletions

diff --git a/src/gallium/drivers/r600/r600_buffer.c b/src/gallium/drivers/r600/r600_buffer.c
index a38c013e16..dc3fc812e1 100644
--- a/src/gallium/drivers/r600/r600_buffer.c
+++ b/src/gallium/drivers/r600/r600_buffer.c
@@ -29,6 +29,7 @@
 #include <util/u_math.h>
 #include <util/u_inlines.h>
 #include <util/u_memory.h>
+#include <util/u_upload_mgr.h>
 #include "state_tracker/drm_driver.h"
 #include "r600_screen.h"
 #include "r600_context.h"
@@ -67,66 +68,69 @@ struct pipe_resource *r600_buffer_create(struct pipe_screen *screen,
 					 const struct pipe_resource *templ)
 {
 	struct r600_screen *rscreen = r600_screen(screen);
-	struct r600_resource *rbuffer;
+	struct r600_resource_buffer *rbuffer;
 	struct radeon_ws_bo *bo;
 	/* XXX We probably want a different alignment for buffers and textures. */
 	unsigned alignment = 4096;
 
-	rbuffer = CALLOC_STRUCT(r600_resource);
+	rbuffer = CALLOC_STRUCT(r600_resource_buffer);
 	if (rbuffer == NULL)
 		return NULL;
 
-	rbuffer->base.b = *templ;
-	pipe_reference_init(&rbuffer->base.b.reference, 1);
-	rbuffer->base.b.screen = screen;
-	rbuffer->base.vtbl = &r600_buffer_vtbl;
-	rbuffer->size = rbuffer->base.b.width0;
-	rbuffer->domain = r600_domain_from_usage(rbuffer->base.b.bind);
-	bo = radeon_ws_bo(rscreen->rw, rbuffer->base.b.width0, alignment, rbuffer->base.b.bind);
+	rbuffer->magic = R600_BUFFER_MAGIC;
+	rbuffer->user_buffer = NULL;
+	rbuffer->num_ranges = 0;
+	rbuffer->r.base.b = *templ;
+	pipe_reference_init(&rbuffer->r.base.b.reference, 1);
+	rbuffer->r.base.b.screen = screen;
+	rbuffer->r.base.vtbl = &r600_buffer_vtbl;
+	rbuffer->r.size = rbuffer->r.base.b.width0;
+	rbuffer->r.domain = r600_domain_from_usage(rbuffer->r.base.b.bind);
+	bo = radeon_ws_bo(rscreen->rw, rbuffer->r.base.b.width0, alignment, rbuffer->r.base.b.bind);
 	if (bo == NULL) {
 		FREE(rbuffer);
 		return NULL;
 	}
-	rbuffer->bo = bo;
-	return &rbuffer->base.b;
+	rbuffer->r.bo = bo;
+	return &rbuffer->r.base.b;
 }
 
 struct pipe_resource *r600_user_buffer_create(struct pipe_screen *screen,
 					      void *ptr, unsigned bytes,
 					      unsigned bind)
 {
-	struct r600_resource *rbuffer;
-	struct r600_screen *rscreen = r600_screen(screen);
-	struct pipe_resource templ;
-	void *data;
+	struct r600_resource_buffer *rbuffer;
 
-	memset(&templ, 0, sizeof(struct pipe_resource));
-	templ.target = PIPE_BUFFER;
-	templ.format = PIPE_FORMAT_R8_UNORM;
-	templ.usage = PIPE_USAGE_IMMUTABLE;
-	templ.bind = bind;
-	templ.width0 = bytes;
-	templ.height0 = 1;
-	templ.depth0 = 1;
-
-	rbuffer = (struct r600_resource*)r600_buffer_create(screen, &templ);
-	if (rbuffer == NULL) {
+	rbuffer = CALLOC_STRUCT(r600_resource_buffer);
+	if (rbuffer == NULL)
 		return NULL;
-	}
-	data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, 0, NULL);
-	memcpy(data, ptr, bytes);
-	radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo);
-	return &rbuffer->base.b;
+
+	rbuffer->magic = R600_BUFFER_MAGIC;
+	pipe_reference_init(&rbuffer->r.base.b.reference, 1);
+	rbuffer->r.base.vtbl = &r600_buffer_vtbl;
+	rbuffer->r.base.b.screen = screen;
+	rbuffer->r.base.b.target = PIPE_BUFFER;
+	rbuffer->r.base.b.format = PIPE_FORMAT_R8_UNORM;
+	rbuffer->r.base.b.usage = PIPE_USAGE_IMMUTABLE;
+	rbuffer->r.base.b.bind = bind;
+	rbuffer->r.base.b.width0 = bytes;
+	rbuffer->r.base.b.height0 = 1;
+	rbuffer->r.base.b.depth0 = 1;
+	rbuffer->r.base.b.flags = 0;
+	rbuffer->num_ranges = 0;
+	rbuffer->r.bo = NULL;
+	rbuffer->user_buffer = ptr;
+	return &rbuffer->r.base.b;
 }
 
 static void r600_buffer_destroy(struct pipe_screen *screen,
 				struct pipe_resource *buf)
 {
-	struct r600_resource *rbuffer = (struct r600_resource*)buf;
+	struct r600_resource_buffer *rbuffer = r600_buffer(buf);
 	struct r600_screen *rscreen = r600_screen(screen);
 
-	if (rbuffer->bo) {
-		radeon_ws_bo_reference(rscreen->rw, &rbuffer->bo, NULL);
+	if (rbuffer->r.bo) {
+		radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL);
 	}
 	FREE(rbuffer);
 }
@@ -134,18 +138,40 @@ static void r600_buffer_destroy(struct pipe_screen *screen,
 static void *r600_buffer_transfer_map(struct pipe_context *pipe,
 				      struct pipe_transfer *transfer)
 {
-	struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource;
+	struct r600_context *rctx = r600_context(pipe);
+	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
 	struct r600_screen *rscreen = r600_screen(pipe->screen);
 	int write = 0;
 	uint8_t *data;
+	int i;
+	boolean flush = FALSE;
+
+	if (rbuffer->user_buffer)
+		return (uint8_t*)rbuffer->user_buffer + transfer->box.x;
 
+	if (transfer->usage & PIPE_TRANSFER_DISCARD) {
+		for (i = 0; i < rbuffer->num_ranges; i++) {
+			if ((transfer->box.x >= rbuffer->ranges[i].start) &&
+			    (transfer->box.x < rbuffer->ranges[i].end))
+				flush = TRUE;
+			
+			if (flush) {
+				radeon_ws_bo_reference(rscreen->rw, &rbuffer->r.bo, NULL);
+				rbuffer->num_ranges = 0;
+				rbuffer->r.bo = radeon_ws_bo(rscreen->rw, 
+							     rbuffer->r.base.b.width0, 0,
+							     rbuffer->r.base.b.bind);
+				break;
+			}
+		}
+	}
 	if (transfer->usage & PIPE_TRANSFER_DONTBLOCK) {
 		/* FIXME */
 	}
 	if (transfer->usage & PIPE_TRANSFER_WRITE) {
 		write = 1;
 	}
-	data = radeon_ws_bo_map(rscreen->rw, rbuffer->bo, transfer->usage, r600_context(pipe));
+	data = radeon_ws_bo_map(rscreen->rw, rbuffer->r.bo, transfer->usage, rctx);
 	if (!data)
 		return NULL;
 
@@ -155,16 +181,39 @@ static void *r600_buffer_transfer_map(struct pipe_context *pipe,
 static void r600_buffer_transfer_unmap(struct pipe_context *pipe,
 					struct pipe_transfer *transfer)
 {
-	struct r600_resource *rbuffer = (struct r600_resource*)transfer->resource;
+	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
 	struct r600_screen *rscreen = r600_screen(pipe->screen);
 
-	radeon_ws_bo_unmap(rscreen->rw, rbuffer->bo);
+	if (rbuffer->r.bo)
+		radeon_ws_bo_unmap(rscreen->rw, rbuffer->r.bo);
 }
 
 static void r600_buffer_transfer_flush_region(struct pipe_context *pipe,
 					      struct pipe_transfer *transfer,
 					      const struct pipe_box *box)
 {
+	struct r600_resource_buffer *rbuffer = r600_buffer(transfer->resource);
+	unsigned i;
+	unsigned offset = transfer->box.x + box->x;
+	unsigned length = box->width;
+
+	assert(box->x + box->width <= transfer->box.width);
+
+	if (rbuffer->user_buffer)
+		return;
+
+	/* mark the range as used */
+	for(i = 0; i < rbuffer->num_ranges; ++i) {
+		if(offset <= rbuffer->ranges[i].end && rbuffer->ranges[i].start <= (offset+box->width)) {
+			rbuffer->ranges[i].start = MIN2(rbuffer->ranges[i].start, offset);
+			rbuffer->ranges[i].end   = MAX2(rbuffer->ranges[i].end, (offset+length));
+			return;
+		}
+	}
+	
+	rbuffer->ranges[rbuffer->num_ranges].start = offset;
+	rbuffer->ranges[rbuffer->num_ranges].end = offset+length;
+	rbuffer->num_ranges++;
 }
 
 unsigned r600_buffer_is_referenced_by_cs(struct pipe_context *context,
@@ -213,3 +262,59 @@ struct u_resource_vtbl r600_buffer_vtbl =
 	r600_buffer_transfer_unmap,		/* transfer_unmap */
 	u_default_transfer_inline_write		/* transfer_inline_write */
 };
+
+int r600_upload_index_buffer(struct r600_context *rctx,
+			     struct r600_draw *draw)
+{
+	struct pipe_resource *upload_buffer = NULL;
+	unsigned index_offset = draw->index_buffer_offset;
+	int ret = 0;
+
+	if (r600_buffer_is_user_buffer(draw->index_buffer)) {
+		ret = u_upload_buffer(rctx->upload_ib,
+				      index_offset,
+				      draw->count * draw->index_size,
+				      draw->index_buffer,
+				      &index_offset,
+				      &upload_buffer);
+		if (ret) {
+			goto done;
+		}
+		draw->index_buffer_offset = index_offset;
+		draw->index_buffer = upload_buffer;
+	}
+
+done:
+	return ret;
+}
+
+int r600_upload_user_buffers(struct r600_context *rctx)
+{
+	enum pipe_error ret = PIPE_OK;
+	int i, nr;
+
+	nr = rctx->vertex_elements->count;
+
+	for (i = 0; i < nr; i++) {
+		struct pipe_vertex_buffer *vb =
+			&rctx->vertex_buffer[rctx->vertex_elements->elements[i].vertex_buffer_index];
+
+		if (r600_buffer_is_user_buffer(vb->buffer)) {
+			struct pipe_resource *upload_buffer = NULL;
+			unsigned offset = 0; /*vb->buffer_offset * 4;*/
+			unsigned size = vb->buffer->width0;
+			unsigned upload_offset;
+			ret = u_upload_buffer(rctx->upload_vb,
+					      offset, size,
+					      vb->buffer,
+					      &upload_offset, &upload_buffer);
+			if (ret)
+				return ret;
+
+			pipe_resource_reference(&vb->buffer, NULL);
+			vb->buffer = upload_buffer;
+			vb->buffer_offset = upload_offset;
+		}
+	}
+	return ret;
+}
diff --git a/src/gallium/drivers/r600/r600_context.c b/src/gallium/drivers/r600/r600_context.c
index cca1e35673..776dc24569 100644
--- a/src/gallium/drivers/r600/r600_context.c
+++ b/src/gallium/drivers/r600/r600_context.c
@@ -28,6 +28,7 @@
 #include <util/u_inlines.h>
 #include <util/u_format.h>
 #include <util/u_memory.h>
+#include <util/u_upload_mgr.h>
 #include <util/u_blitter.h>
 #include "r600_screen.h"
 #include "r600_context.h"
@@ -56,6 +57,9 @@ static void r600_destroy_context(struct pipe_context *context)
 	free(rctx->vs_constant);
 	free(rctx->vs_resource);
 
+	u_upload_destroy(rctx->upload_vb);
+	u_upload_destroy(rctx->upload_ib);
+
 	radeon_ctx_fini(rctx->ctx);
 	FREE(rctx);
 }
@@ -66,6 +70,10 @@ void r600_flush(struct pipe_context *ctx, unsigned flags,
 	struct r600_context *rctx = r600_context(ctx);
 	struct r600_query *rquery = NULL;
 
+	/* flush upload buffers */
+	u_upload_flush(rctx->upload_vb);
+	u_upload_flush(rctx->upload_ib);
+
 	/* suspend queries */
 	r600_queries_suspend(ctx);
 
@@ -123,25 +131,37 @@ struct pipe_context *r600_create_context(struct pipe_screen *screen, void *priv)
 
 	rctx->vtbl->init_config(rctx);
 
+	rctx->upload_ib = u_upload_create(&rctx->context, 32 * 1024, 16,
+					  PIPE_BIND_INDEX_BUFFER);
+	if (rctx->upload_ib == NULL) {
+		goto out_free;
+	}
+
+	rctx->upload_vb = u_upload_create(&rctx->context, 128 * 1024, 16,
+					  PIPE_BIND_VERTEX_BUFFER);
+	if (rctx->upload_vb == NULL) {
+		goto out_free;
+	}
+
 	rctx->vs_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state));
 	if (!rctx->vs_constant) {
-		FREE(rctx);
-		return NULL;
+		goto out_free;
 	}
 
 	rctx->ps_constant = (struct radeon_state *)calloc(R600_MAX_CONSTANT, sizeof(struct radeon_state));
 	if (!rctx->ps_constant) {
-		FREE(rctx);
-		return NULL;
+		goto out_free;
 	}
 
 	rctx->vs_resource = (struct radeon_state *)calloc(R600_MAX_RESOURCE, sizeof(struct radeon_state));
 	if (!rctx->vs_resource) {
-		FREE(rctx);
-		return NULL;
+		goto out_free;
 	}						   
 
 	rctx->ctx = radeon_ctx_init(rscreen->rw);
 	radeon_draw_init(&rctx->draw, rscreen->rw);
 	return &rctx->context;
+ out_free:
+	FREE(rctx);
+	return NULL;
 }
diff --git a/src/gallium/drivers/r600/r600_context.h b/src/gallium/drivers/r600/r600_context.h
index f82c8f82fe..3107f189c7 100644
--- a/src/gallium/drivers/r600/r600_context.h
+++ b/src/gallium/drivers/r600/r600_context.h
@@ -34,6 +34,8 @@
 #include "radeon.h"
 #include "r600_shader.h"
 
+struct u_upload_mgr;
+
 #define R600_QUERY_STATE_STARTED	(1 << 0)
 #define R600_QUERY_STATE_ENDED		(1 << 1)
 #define R600_QUERY_STATE_SUSPENDED	(1 << 2)
@@ -249,6 +251,12 @@ struct r600_context {
 	struct pipe_index_buffer	index_buffer;
 	struct pipe_blend_color		blend_color;
 	struct list_head		query_list;
+
+	/* upload managers */
+	struct u_upload_mgr *upload_vb;
+	struct u_upload_mgr *upload_ib;
+	bool any_user_vbs;
+
 };
 
 /* Convenience cast wrapper. */
@@ -306,4 +314,8 @@ void eg_set_constant_buffer(struct pipe_context *ctx,
 			    uint shader, uint index,
 			    struct pipe_resource *buffer);
 
+int r600_upload_index_buffer(struct r600_context *rctx,
+                             struct r600_draw *draw);
+int r600_upload_user_buffers(struct r600_context *rctx);
+
 #endif
diff --git a/src/gallium/drivers/r600/r600_draw.c b/src/gallium/drivers/r600/r600_draw.c
index 669c9b4cdb..5480ca002d 100644
--- a/src/gallium/drivers/r600/r600_draw.c
+++ b/src/gallium/drivers/r600/r600_draw.c
@@ -126,6 +126,11 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 
 	memset(&draw, 0, sizeof(draw));
 
+	if (rctx->any_user_vbs) {
+		r600_upload_user_buffers(rctx);
+		rctx->any_user_vbs = false;
+	}
+
 	draw.ctx = ctx;
 	draw.mode = info->mode;
 	draw.start = info->start;
@@ -139,8 +144,7 @@ void r600_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info)
 
 		assert(rctx->index_buffer.offset %
 				rctx->index_buffer.index_size == 0);
-		draw.start += rctx->index_buffer.offset /
-			rctx->index_buffer.index_size;
+		r600_upload_index_buffer(rctx, &draw);
 	}
 	else {
 		draw.index_size = 0;
diff --git a/src/gallium/drivers/r600/r600_resource.h b/src/gallium/drivers/r600/r600_resource.h
index ff05afbc30..9608a5a623 100644
--- a/src/gallium/drivers/r600/r600_resource.h
+++ b/src/gallium/drivers/r600/r600_resource.h
@@ -75,4 +75,35 @@ struct pipe_resource *r600_texture_from_handle(struct pipe_screen *screen,
 						const struct pipe_resource *base,
 						struct winsys_handle *whandle);
 
+#define R600_BUFFER_MAGIC 0xabcd1600
+#define R600_BUFFER_MAX_RANGES 32
+
+struct r600_buffer_range {
+	uint32_t start;
+	uint32_t end;
+};
+
+struct r600_resource_buffer {
+	struct r600_resource r;
+	uint32_t magic;
+	void *user_buffer;
+	struct r600_buffer_range ranges[R600_BUFFER_MAX_RANGES];
+	unsigned num_ranges;
+};
+
+/* r600_buffer */
+static INLINE struct r600_resource_buffer *r600_buffer(struct pipe_resource *buffer)
+{
+	if (buffer) {
+		assert(((struct r600_resource_buffer *)buffer)->magic == R600_BUFFER_MAGIC);
+		return (struct r600_resource_buffer *)buffer;
+    }
+    return NULL;
+}
+
+static INLINE boolean r600_buffer_is_user_buffer(struct pipe_resource *buffer)
+{
+    return r600_buffer(buffer)->user_buffer ? true : false;
+}
+
 #endif
diff --git a/src/gallium/drivers/r600/r600_state.c b/src/gallium/drivers/r600/r600_state.c
index 5d6236206f..4dcdc492fc 100644
--- a/src/gallium/drivers/r600/r600_state.c
+++ b/src/gallium/drivers/r600/r600_state.c
@@ -437,6 +437,7 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx,
 {
 	struct r600_context *rctx = r600_context(ctx);
 	unsigned i;
+	boolean any_user_buffers = FALSE;
 
 	for (i = 0; i < rctx->nvertex_buffer; i++) {
 		pipe_resource_reference(&rctx->vertex_buffer[i].buffer, NULL);
@@ -444,8 +445,11 @@ static void r600_set_vertex_buffers(struct pipe_context *ctx,
 	memcpy(rctx->vertex_buffer, buffers, sizeof(struct pipe_vertex_buffer) * count);
 	for (i = 0; i < count; i++) {
 		rctx->vertex_buffer[i].buffer = NULL;
+		if (r600_buffer_is_user_buffer(buffers[i].buffer))
+			any_user_buffers = TRUE;
 		pipe_resource_reference(&rctx->vertex_buffer[i].buffer, buffers[i].buffer);
 	}
+	rctx->any_user_vbs = any_user_buffers;
 	rctx->nvertex_buffer = count;
 }