Merge remote branch 'main/master' into radeon-rewrite

Conflicts:
	src/mesa/drivers/dri/r300/r300_cmdbuf.c
	src/mesa/drivers/dri/r300/r300_state.c
	src/mesa/drivers/dri/r300/r300_swtcl.c
	src/mesa/drivers/dri/r300/radeon_ioctl.c
	src/mesa/drivers/dri/radeon/radeon_screen.c

16 files changed, 268 insertions, 177 deletions

diff --git a/src/gallium/drivers/nv40/Makefile b/src/gallium/drivers/nv40/Makefile
index 9c8eadf7e4..0ecae2b491 100644
--- a/src/gallium/drivers/nv40/Makefile
+++ b/src/gallium/drivers/nv40/Makefile
@@ -3,7 +3,7 @@ include $(TOP)/configs/current
 
 LIBNAME = nv40
 
-DRIVER_SOURCES = \
+C_SOURCES = \
 	nv40_clear.c \
 	nv40_context.c \
 	nv40_draw.c \
@@ -22,16 +22,8 @@ DRIVER_SOURCES = \
 	nv40_state_viewport.c \
 	nv40_state_zsa.c \
 	nv40_surface.c \
+	nv40_transfer.c \
 	nv40_vbo.c \
 	nv40_vertprog.c
 
-C_SOURCES = \
-	$(COMMON_SOURCES) \
-	$(DRIVER_SOURCES)
-
-ASM_SOURCES = 
-
 include ../../Makefile.template
-
-symlinks:
-
diff --git a/src/gallium/drivers/nv40/nv40_fragprog.c b/src/gallium/drivers/nv40/nv40_fragprog.c
index 91dcbebda0..16e40889ec 100644
--- a/src/gallium/drivers/nv40/nv40_fragprog.c
+++ b/src/gallium/drivers/nv40/nv40_fragprog.c
@@ -917,6 +917,7 @@ nv40_fragprog_validate(struct nv40_context *nv40)
 	struct nv40_fragment_program *fp = nv40->fragprog;
 	struct pipe_buffer *constbuf =
 		nv40->constbuf[PIPE_SHADER_FRAGMENT];
+	struct pipe_screen *screen = nv40->pipe.screen;
 	struct pipe_winsys *ws = nv40->pipe.winsys;
 	struct nouveau_stateobj *so;
 	boolean new_consts = FALSE;
@@ -932,7 +933,7 @@ nv40_fragprog_validate(struct nv40_context *nv40)
 		return FALSE;
 	}
 
-	fp->buffer = ws->buffer_create(ws, 0x100, 0, fp->insn_len * 4);
+	fp->buffer = screen->buffer_create(screen, 0x100, 0, fp->insn_len * 4);
 	nv40_fragprog_upload(nv40, fp);
 
 	so = so_new(4, 1);
@@ -943,6 +944,7 @@ nv40_fragprog_validate(struct nv40_context *nv40)
 	so_method(so, nv40->screen->curie, NV40TCL_FP_CONTROL, 1);
 	so_data  (so, fp->fp_control);
 	so_ref(so, &fp->so);
+	so_ref(NULL, &so);
 
 update_constants:
 	if (fp->nr_consts) {
diff --git a/src/gallium/drivers/nv40/nv40_fragtex.c b/src/gallium/drivers/nv40/nv40_fragtex.c
index 0227d22620..eb3002dc05 100644
--- a/src/gallium/drivers/nv40/nv40_fragtex.c
+++ b/src/gallium/drivers/nv40/nv40_fragtex.c
@@ -151,6 +151,7 @@ nv40_fragtex_validate(struct nv40_context *nv40)
 
 		so = nv40_fragtex_build(nv40, unit);
 		so_ref(so, &nv40->state.hw[NV40_STATE_FRAGTEX0 + unit]);
+		so_ref(NULL, &so);
 		state->dirty |= (1ULL << (NV40_STATE_FRAGTEX0 + unit));
 	}
 
diff --git a/src/gallium/drivers/nv40/nv40_miptree.c b/src/gallium/drivers/nv40/nv40_miptree.c
index e38b1e7f5c..abadca8c93 100644
--- a/src/gallium/drivers/nv40/nv40_miptree.c
+++ b/src/gallium/drivers/nv40/nv40_miptree.c
@@ -67,7 +67,6 @@ nv40_miptree_layout(struct nv40_miptree *mt)
 static struct pipe_texture *
 nv40_miptree_create(struct pipe_screen *pscreen, const struct pipe_texture *pt)
 {
-	struct pipe_winsys *ws = pscreen->winsys;
 	struct nv40_miptree *mt;
 	unsigned buf_usage = PIPE_BUFFER_USAGE_PIXEL |
 	                     NOUVEAU_BUFFER_USAGE_TEXTURE;
@@ -76,10 +75,8 @@ nv40_miptree_create(struct pipe_screen *pscreen, const struct pipe_texture *pt)
 	if (!mt)
 		return NULL;
 	mt->base = *pt;
-	mt->base.refcount = 1;
+	pipe_reference_init(&mt->base.reference, 1);
 	mt->base.screen = pscreen;
-	mt->shadow_tex = NULL;
-	mt->shadow_surface = NULL;
 
 	/* Swizzled textures must be POT */
 	if (pt->width[0] & (pt->width[0] - 1) ||
@@ -114,7 +111,7 @@ nv40_miptree_create(struct pipe_screen *pscreen, const struct pipe_texture *pt)
 
 	nv40_miptree_layout(mt);
 
-	mt->buffer = ws->buffer_create(ws, 256, buf_usage, mt->total_size);
+	mt->buffer = pscreen->buffer_create(pscreen, 256, buf_usage, mt->total_size);
 	if (!mt->buffer) {
 		FREE(mt);
 		return NULL;
@@ -139,38 +136,27 @@ nv40_miptree_blanket(struct pipe_screen *pscreen, const struct pipe_texture *pt,
 		return NULL;
 
 	mt->base = *pt;
-	mt->base.refcount = 1;
+	pipe_reference_init(&mt->base.reference, 1);
 	mt->base.screen = pscreen;
 	mt->level[0].pitch = stride[0];
 	mt->level[0].image_offset = CALLOC(1, sizeof(unsigned));
 
-	pipe_buffer_reference(pscreen, &mt->buffer, pb);
+	pipe_buffer_reference(&mt->buffer, pb);
 	return &mt->base;
 }
 
 static void
-nv40_miptree_release(struct pipe_screen *pscreen, struct pipe_texture **ppt)
+nv40_miptree_destroy(struct pipe_texture *pt)
 {
-	struct pipe_texture *pt = *ppt;
 	struct nv40_miptree *mt = (struct nv40_miptree *)pt;
 	int l;
 
-	*ppt = NULL;
-	if (--pt->refcount)
-		return;
-
-	pipe_buffer_reference(pscreen, &mt->buffer, NULL);
+	pipe_buffer_reference(&mt->buffer, NULL);
 	for (l = 0; l <= pt->last_level; l++) {
 		if (mt->level[l].image_offset)
 			FREE(mt->level[l].image_offset);
 	}
 
-	if (mt->shadow_tex) {
-		if (mt->shadow_surface)
-			pscreen->tex_surface_release(pscreen, &mt->shadow_surface);
-		nv40_miptree_release(pscreen, &mt->shadow_tex);
-	}
-
 	FREE(mt);
 }
 
@@ -180,48 +166,38 @@ nv40_miptree_surface_new(struct pipe_screen *pscreen, struct pipe_texture *pt,
 			 unsigned flags)
 {
 	struct nv40_miptree *mt = (struct nv40_miptree *)pt;
-	struct pipe_surface *ps;
+	struct nv04_surface *ns;
 
-	ps = CALLOC_STRUCT(pipe_surface);
-	if (!ps)
+	ns = CALLOC_STRUCT(nv04_surface);
+	if (!ns)
 		return NULL;
-	pipe_texture_reference(&ps->texture, pt);
-	ps->format = pt->format;
-	ps->width = pt->width[level];
-	ps->height = pt->height[level];
-	ps->block = pt->block;
-	ps->nblocksx = pt->nblocksx[level];
-	ps->nblocksy = pt->nblocksy[level];
-	ps->stride = mt->level[level].pitch;
-	ps->usage = flags;
-	ps->status = PIPE_SURFACE_STATUS_DEFINED;
-	ps->refcount = 1;
-	ps->face = face;
-	ps->level = level;
-	ps->zslice = zslice;
+	pipe_texture_reference(&ns->base.texture, pt);
+	ns->base.format = pt->format;
+	ns->base.width = pt->width[level];
+	ns->base.height = pt->height[level];
+	ns->base.usage = flags;
+	ns->base.status = PIPE_SURFACE_STATUS_DEFINED;
+	pipe_reference_init(&ns->base.reference, 1);
+	ns->base.face = face;
+	ns->base.level = level;
+	ns->base.zslice = zslice;
+	ns->pitch = mt->level[level].pitch;
 
 	if (pt->target == PIPE_TEXTURE_CUBE) {
-		ps->offset = mt->level[level].image_offset[face];
+		ns->base.offset = mt->level[level].image_offset[face];
 	} else
 	if (pt->target == PIPE_TEXTURE_3D) {
-		ps->offset = mt->level[level].image_offset[zslice];
+		ns->base.offset = mt->level[level].image_offset[zslice];
 	} else {
-		ps->offset = mt->level[level].image_offset[0];
+		ns->base.offset = mt->level[level].image_offset[0];
 	}
 
-	return ps;
+	return &ns->base;
 }
 
 static void
-nv40_miptree_surface_del(struct pipe_screen *pscreen,
-			 struct pipe_surface **psurface)
+nv40_miptree_surface_del(struct pipe_surface *ps)
 {
-	struct pipe_surface *ps = *psurface;
-
-	*psurface = NULL;
-	if (--ps->refcount > 0)
-		return;
-
 	pipe_texture_reference(&ps->texture, NULL);
 	FREE(ps);
 }
@@ -231,8 +207,8 @@ nv40_screen_init_miptree_functions(struct pipe_screen *pscreen)
 {
 	pscreen->texture_create = nv40_miptree_create;
 	pscreen->texture_blanket = nv40_miptree_blanket;
-	pscreen->texture_release = nv40_miptree_release;
+	pscreen->texture_destroy = nv40_miptree_destroy;
 	pscreen->get_tex_surface = nv40_miptree_surface_new;
-	pscreen->tex_surface_release = nv40_miptree_surface_del;
+	pscreen->tex_surface_destroy = nv40_miptree_surface_del;
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_screen.c b/src/gallium/drivers/nv40/nv40_screen.c
index 2372bc8441..0d4baefaea 100644
--- a/src/gallium/drivers/nv40/nv40_screen.c
+++ b/src/gallium/drivers/nv40/nv40_screen.c
@@ -144,81 +144,6 @@ nv40_surface_buffer(struct pipe_surface *surf)
 	return mt->buffer;
 }
 
-static void *
-nv40_surface_map(struct pipe_screen *screen, struct pipe_surface *surface,
-		 unsigned flags )
-{
-	struct pipe_winsys	*ws = screen->winsys;
-	struct pipe_surface	*surface_to_map;
-	void			*map;
-
-	if (!(surface->texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR)) {
-		struct nv40_miptree *mt = (struct nv40_miptree *)surface->texture;
-
-		if (!mt->shadow_tex) {
-			unsigned old_tex_usage = surface->texture->tex_usage;
-			surface->texture->tex_usage = NOUVEAU_TEXTURE_USAGE_LINEAR |
-			                              PIPE_TEXTURE_USAGE_DYNAMIC;
-			mt->shadow_tex = screen->texture_create(screen, surface->texture);
-			surface->texture->tex_usage = old_tex_usage;
-
-			assert(mt->shadow_tex->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR);
-		}
-
-		mt->shadow_surface = screen->get_tex_surface
-		(
-			screen, mt->shadow_tex,
-			surface->face, surface->level, surface->zslice,
-			surface->usage
-		);
-
-		surface_to_map = mt->shadow_surface;
-	}
-	else
-		surface_to_map = surface;
-
-	assert(surface_to_map);
-	map = ws->buffer_map(ws, nv40_surface_buffer(surface_to_map), flags);
-	if (!map)
-		return NULL;
-
-	return map + surface_to_map->offset;
-}
-
-static void
-nv40_surface_unmap(struct pipe_screen *screen, struct pipe_surface *surface)
-{
-	struct pipe_winsys	*ws = screen->winsys;
-	struct pipe_surface	*surface_to_unmap;
-
-	/* TODO: Copy from shadow just before push buffer is flushed instead.
-	         There are probably some programs that map/unmap excessively
-	         before rendering. */
-	if (!(surface->texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR)) {
-		struct nv40_miptree *mt = (struct nv40_miptree *)surface->texture;
-
-		assert(mt->shadow_tex);
-
-		surface_to_unmap = mt->shadow_surface;
-	}
-	else
-		surface_to_unmap = surface;
-
-	assert(surface_to_unmap);
-
-	ws->buffer_unmap(ws, nv40_surface_buffer(surface_to_unmap));
-
-	if (surface_to_unmap != surface) {
-		struct nv40_screen *nvscreen = nv40_screen(screen);
-
-		nvscreen->eng2d->copy(nvscreen->eng2d, surface, 0, 0,
-		                      surface_to_unmap, 0, 0,
-		                      surface->width, surface->height);
-
-		screen->tex_surface_release(screen, &surface_to_unmap);
-	}
-}
-
 static void
 nv40_screen_destroy(struct pipe_screen *pscreen)
 {
@@ -240,7 +165,7 @@ nv40_screen_create(struct pipe_winsys *ws, struct nouveau_winsys *nvws)
 {
 	struct nv40_screen *screen = CALLOC_STRUCT(nv40_screen);
 	struct nouveau_stateobj *so;
-	unsigned curie_class;
+	unsigned curie_class = 0;
 	unsigned chipset = nvws->channel->device->chipset;
 	int ret;
 
@@ -265,8 +190,6 @@ nv40_screen_create(struct pipe_winsys *ws, struct nouveau_winsys *nvws)
 		if (NV6X_GRCLASS4497_CHIPSETS & (1 << (chipset & 0x0f)))
 			curie_class = NV44TCL;
 		break;
-	default:
-		break;
 	}
 
 	if (!curie_class) {
@@ -372,10 +295,8 @@ nv40_screen_create(struct pipe_winsys *ws, struct nouveau_winsys *nvws)
 
 	screen->pipe.is_format_supported = nv40_screen_surface_format_supported;
 
-	screen->pipe.surface_map = nv40_surface_map;
-	screen->pipe.surface_unmap = nv40_surface_unmap;
-
 	nv40_screen_init_miptree_functions(&screen->pipe);
+	nv40_screen_init_transfer_functions(&screen->pipe);
 	u_simple_screen_init(&screen->pipe);
 
 	return &screen->pipe;
diff --git a/src/gallium/drivers/nv40/nv40_screen.h b/src/gallium/drivers/nv40/nv40_screen.h
index 4500aa0e5c..7b503bd207 100644
--- a/src/gallium/drivers/nv40/nv40_screen.h
+++ b/src/gallium/drivers/nv40/nv40_screen.h
@@ -34,4 +34,7 @@ nv40_screen(struct pipe_screen *screen)
 	return (struct nv40_screen *)screen;
 }
 
+void
+nv40_screen_init_transfer_functions(struct pipe_screen *pscreen);
+
 #endif
diff --git a/src/gallium/drivers/nv40/nv40_state.c b/src/gallium/drivers/nv40/nv40_state.c
index 2eff25aa83..c3ee4d2345 100644
--- a/src/gallium/drivers/nv40/nv40_state.c
+++ b/src/gallium/drivers/nv40/nv40_state.c
@@ -52,6 +52,7 @@ nv40_blend_state_create(struct pipe_context *pipe,
 	so_data  (so, cso->dither ? 1 : 0);
 
 	so_ref(so, &bso->so);
+	so_ref(NULL, &so);
 	bso->pipe = *cso;
 	return (void *)bso;
 }
@@ -414,6 +415,7 @@ nv40_rasterizer_state_create(struct pipe_context *pipe,
 	}
 
 	so_ref(so, &rsso->so);
+	so_ref(NULL, &so);
 	rsso->pipe = *cso;
 	return (void *)rsso;
 }
@@ -487,6 +489,7 @@ nv40_depth_stencil_alpha_state_create(struct pipe_context *pipe,
 	}
 
 	so_ref(so, &zsaso->so);
+	so_ref(NULL, &so);
 	zsaso->pipe = *cso;
 	return (void *)zsaso;
 }
diff --git a/src/gallium/drivers/nv40/nv40_state.h b/src/gallium/drivers/nv40/nv40_state.h
index 9c55903ae3..8a9d8c8fdf 100644
--- a/src/gallium/drivers/nv40/nv40_state.h
+++ b/src/gallium/drivers/nv40/nv40_state.h
@@ -79,9 +79,6 @@ struct nv40_miptree {
 	struct pipe_buffer *buffer;
 	uint total_size;
 
-	struct pipe_texture *shadow_tex;
-	struct pipe_surface *shadow_surface;
-
 	struct {
 		uint pitch;
 		uint *image_offset;
diff --git a/src/gallium/drivers/nv40/nv40_state_blend.c b/src/gallium/drivers/nv40/nv40_state_blend.c
index 95e6d7394f..8cd05ce66e 100644
--- a/src/gallium/drivers/nv40/nv40_state_blend.c
+++ b/src/gallium/drivers/nv40/nv40_state_blend.c
@@ -28,6 +28,7 @@ nv40_state_blend_colour_validate(struct nv40_context *nv40)
 		       (float_to_ubyte(bcol->color[2]) <<  0)));
 
 	so_ref(so, &nv40->state.hw[NV40_STATE_BCOL]);
+	so_ref(NULL, &so);
 	return TRUE;
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_state_fb.c b/src/gallium/drivers/nv40/nv40_state_fb.c
index 454abad31f..be618a306b 100644
--- a/src/gallium/drivers/nv40/nv40_state_fb.c
+++ b/src/gallium/drivers/nv40/nv40_state_fb.c
@@ -12,7 +12,7 @@ static boolean
 nv40_state_framebuffer_validate(struct nv40_context *nv40)
 {
 	struct pipe_framebuffer_state *fb = &nv40->framebuffer;
-	struct pipe_surface *rt[4], *zeta;
+	struct nv04_surface *rt[4], *zeta;
 	uint32_t rt_enable, rt_format;
 	int i, colour_format = 0, zeta_format = 0;
 	struct nouveau_stateobj *so = so_new(64, 10);
@@ -27,7 +27,7 @@ nv40_state_framebuffer_validate(struct nv40_context *nv40)
 		} else {
 			colour_format = fb->cbufs[i]->format;
 			rt_enable |= (NV40TCL_RT_ENABLE_COLOR0 << i);
-			rt[i] = fb->cbufs[i];
+			rt[i] = (struct nv04_surface *)fb->cbufs[i];
 		}
 	}
 
@@ -37,13 +37,13 @@ nv40_state_framebuffer_validate(struct nv40_context *nv40)
 
 	if (fb->zsbuf) {
 		zeta_format = fb->zsbuf->format;
-		zeta = fb->zsbuf;
+		zeta = (struct nv04_surface *)fb->zsbuf;
 	}
 
-	if (!(rt[0]->texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR)) {
+	if (!(rt[0]->base.texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR)) {
 		assert(!(fb->width & (fb->width - 1)) && !(fb->height & (fb->height - 1)));
 		for (i = 1; i < fb->nr_cbufs; i++)
-			assert(!(rt[i]->texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR));
+			assert(!(rt[i]->base.texture->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR));
 
 		rt_format = NV40TCL_RT_FORMAT_TYPE_SWIZZLED |
 		            log2i(fb->width) << NV40TCL_RT_FORMAT_LOG2_WIDTH_SHIFT |
@@ -78,60 +78,60 @@ nv40_state_framebuffer_validate(struct nv40_context *nv40)
 
 	if (rt_enable & NV40TCL_RT_ENABLE_COLOR0) {
 		so_method(so, nv40->screen->curie, NV40TCL_DMA_COLOR0, 1);
-		so_reloc (so, nv40_surface_buffer(rt[0]), 0, rt_flags | NOUVEAU_BO_OR,
+		so_reloc (so, nv40_surface_buffer(&rt[0]->base), 0, rt_flags | NOUVEAU_BO_OR,
 			  nv40->nvws->channel->vram->handle,
 			  nv40->nvws->channel->gart->handle);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR0_PITCH, 2);
-		so_data  (so, rt[0]->stride);
-		so_reloc (so, nv40_surface_buffer(rt[0]), rt[0]->offset, rt_flags |
+		so_data  (so, rt[0]->pitch);
+		so_reloc (so, nv40_surface_buffer(&rt[0]->base), rt[0]->base.offset, rt_flags |
 			  NOUVEAU_BO_LOW, 0, 0);
 	}
 
 	if (rt_enable & NV40TCL_RT_ENABLE_COLOR1) {
 		so_method(so, nv40->screen->curie, NV40TCL_DMA_COLOR1, 1);
-		so_reloc (so, nv40_surface_buffer(rt[1]), 0, rt_flags | NOUVEAU_BO_OR,
+		so_reloc (so, nv40_surface_buffer(&rt[1]->base), 0, rt_flags | NOUVEAU_BO_OR,
 			  nv40->nvws->channel->vram->handle,
 			  nv40->nvws->channel->gart->handle);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR1_OFFSET, 2);
-		so_reloc (so, nv40_surface_buffer(rt[1]), rt[1]->offset, rt_flags |
+		so_reloc (so, nv40_surface_buffer(&rt[1]->base), rt[1]->base.offset, rt_flags |
 			  NOUVEAU_BO_LOW, 0, 0);
-		so_data  (so, rt[1]->stride);
+		so_data  (so, rt[1]->pitch);
 	}
 
 	if (rt_enable & NV40TCL_RT_ENABLE_COLOR2) {
 		so_method(so, nv40->screen->curie, NV40TCL_DMA_COLOR2, 1);
-		so_reloc (so, nv40_surface_buffer(rt[2]), 0, rt_flags | NOUVEAU_BO_OR,
+		so_reloc (so, nv40_surface_buffer(&rt[2]->base), 0, rt_flags | NOUVEAU_BO_OR,
 			  nv40->nvws->channel->vram->handle,
 			  nv40->nvws->channel->gart->handle);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR2_OFFSET, 1);
-		so_reloc (so, nv40_surface_buffer(rt[2]), rt[2]->offset, rt_flags |
+		so_reloc (so, nv40_surface_buffer(&rt[2]->base), rt[2]->base.offset, rt_flags |
 			  NOUVEAU_BO_LOW, 0, 0);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR2_PITCH, 1);
-		so_data  (so, rt[2]->stride);
+		so_data  (so, rt[2]->pitch);
 	}
 
 	if (rt_enable & NV40TCL_RT_ENABLE_COLOR3) {
 		so_method(so, nv40->screen->curie, NV40TCL_DMA_COLOR3, 1);
-		so_reloc (so, nv40_surface_buffer(rt[3]), 0, rt_flags | NOUVEAU_BO_OR,
+		so_reloc (so, nv40_surface_buffer(&rt[3]->base), 0, rt_flags | NOUVEAU_BO_OR,
 			  nv40->nvws->channel->vram->handle,
 			  nv40->nvws->channel->gart->handle);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR3_OFFSET, 1);
-		so_reloc (so, nv40_surface_buffer(rt[3]), rt[3]->offset, rt_flags |
+		so_reloc (so, nv40_surface_buffer(&rt[3]->base), rt[3]->base.offset, rt_flags |
 			  NOUVEAU_BO_LOW, 0, 0);
 		so_method(so, nv40->screen->curie, NV40TCL_COLOR3_PITCH, 1);
-		so_data  (so, rt[3]->stride);
+		so_data  (so, rt[3]->pitch);
 	}
 
 	if (zeta_format) {
 		so_method(so, nv40->screen->curie, NV40TCL_DMA_ZETA, 1);
-		so_reloc (so, nv40_surface_buffer(zeta), 0, rt_flags | NOUVEAU_BO_OR,
+		so_reloc (so, nv40_surface_buffer(&zeta->base), 0, rt_flags | NOUVEAU_BO_OR,
 			  nv40->nvws->channel->vram->handle,
 			  nv40->nvws->channel->gart->handle);
 		so_method(so, nv40->screen->curie, NV40TCL_ZETA_OFFSET, 1);
-		so_reloc (so, nv40_surface_buffer(zeta), zeta->offset, rt_flags |
+		so_reloc (so, nv40_surface_buffer(&zeta->base), zeta->base.offset, rt_flags |
 			  NOUVEAU_BO_LOW, 0, 0);
 		so_method(so, nv40->screen->curie, NV40TCL_ZETA_PITCH, 1);
-		so_data  (so, zeta->stride);
+		so_data  (so, zeta->pitch);
 	}
 
 	so_method(so, nv40->screen->curie, NV40TCL_RT_ENABLE, 1);
@@ -150,6 +150,7 @@ nv40_state_framebuffer_validate(struct nv40_context *nv40)
 	so_data  (so, (1 << 12) | h);
 
 	so_ref(so, &nv40->state.hw[NV40_STATE_FB]);
+	so_ref(NULL, &so);
 	return TRUE;
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_state_scissor.c b/src/gallium/drivers/nv40/nv40_state_scissor.c
index 285239ef41..cf58d33906 100644
--- a/src/gallium/drivers/nv40/nv40_state_scissor.c
+++ b/src/gallium/drivers/nv40/nv40_state_scissor.c
@@ -23,6 +23,7 @@ nv40_state_scissor_validate(struct nv40_context *nv40)
 	}
 
 	so_ref(so, &nv40->state.hw[NV40_STATE_SCISSOR]);
+	so_ref(NULL, &so);
 	return TRUE;
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_state_viewport.c b/src/gallium/drivers/nv40/nv40_state_viewport.c
index 869a55b405..665d2d5fca 100644
--- a/src/gallium/drivers/nv40/nv40_state_viewport.c
+++ b/src/gallium/drivers/nv40/nv40_state_viewport.c
@@ -7,7 +7,8 @@ nv40_state_viewport_validate(struct nv40_context *nv40)
 	struct nouveau_stateobj *so;
 	unsigned bypass;
 
-	if (nv40->render_mode == HW && !nv40->rasterizer->pipe.bypass_clipping)
+	if (nv40->render_mode == HW &&
+	    !nv40->rasterizer->pipe.bypass_vs_clip_and_viewport)
 		bypass = 0;
 	else
 		bypass = 1;
@@ -55,6 +56,7 @@ nv40_state_viewport_validate(struct nv40_context *nv40)
 	}
 
 	so_ref(so, &nv40->state.hw[NV40_STATE_VIEWPORT]);
+	so_ref(NULL, &so);
 	return TRUE;
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_surface.c b/src/gallium/drivers/nv40/nv40_surface.c
index c4a5fb20d9..1a849da32d 100644
--- a/src/gallium/drivers/nv40/nv40_surface.c
+++ b/src/gallium/drivers/nv40/nv40_surface.c
@@ -33,7 +33,7 @@
 #include "util/u_tile.h"
 
 static void
-nv40_surface_copy(struct pipe_context *pipe, boolean do_flip,
+nv40_surface_copy(struct pipe_context *pipe,
 		  struct pipe_surface *dest, unsigned destx, unsigned desty,
 		  struct pipe_surface *src, unsigned srcx, unsigned srcy,
 		  unsigned width, unsigned height)
@@ -41,15 +41,6 @@ nv40_surface_copy(struct pipe_context *pipe, boolean do_flip,
 	struct nv40_context *nv40 = nv40_context(pipe);
 	struct nv04_surface_2d *eng2d = nv40->screen->eng2d;
 
-	if (do_flip) {
-		desty += height;
-		while (height--) {
-			eng2d->copy(eng2d, dest, destx, desty--, src,
-				    srcx, srcy++, width, 1);
-		}
-		return;
-	}
-
 	eng2d->copy(eng2d, dest, destx, desty, src, srcx, srcy, width, height);
 }
 
diff --git a/src/gallium/drivers/nv40/nv40_transfer.c b/src/gallium/drivers/nv40/nv40_transfer.c
new file mode 100644
index 0000000000..728e8b5674
--- /dev/null
+++ b/src/gallium/drivers/nv40/nv40_transfer.c
@@ -0,0 +1,196 @@
+#include <pipe/p_state.h>
+#include <pipe/p_defines.h>
+#include <pipe/p_inlines.h>
+#include <util/u_memory.h>
+#include <nouveau/nouveau_winsys.h>
+#include "nv40_context.h"
+#include "nv40_screen.h"
+#include "nv40_state.h"
+
+struct nv40_transfer {
+	struct pipe_transfer base;
+	struct pipe_surface *surface;
+	bool direct;
+};
+
+static unsigned nv40_usage_tx_to_buf(unsigned tx_usage)
+{
+	switch (tx_usage) {
+		case PIPE_TRANSFER_READ:
+			return PIPE_BUFFER_USAGE_CPU_READ;
+		case PIPE_TRANSFER_WRITE:
+			return PIPE_BUFFER_USAGE_CPU_WRITE;
+		case PIPE_TRANSFER_READ_WRITE:
+			return PIPE_BUFFER_USAGE_CPU_READ_WRITE;
+		default:
+			assert(0);
+	}
+
+	return -1;
+}
+
+static void
+nv40_compatible_transfer_tex(struct pipe_texture *pt, unsigned level,
+                             struct pipe_texture *template)
+{
+	memset(template, 0, sizeof(struct pipe_texture));
+	template->target = pt->target;
+	template->format = pt->format;
+	template->width[0] = pt->width[level];
+	template->height[0] = pt->height[level];
+	template->depth[0] = 1;
+	template->block = pt->block;
+	template->nblocksx[0] = pt->nblocksx[level];
+	template->nblocksy[0] = pt->nblocksx[level];
+	template->last_level = 0;
+	template->compressed = pt->compressed;
+	template->nr_samples = pt->nr_samples;
+
+	template->tex_usage = PIPE_TEXTURE_USAGE_DYNAMIC |
+	                      NOUVEAU_TEXTURE_USAGE_LINEAR;
+}
+
+static struct pipe_transfer *
+nv40_transfer_new(struct pipe_screen *pscreen, struct pipe_texture *pt,
+		  unsigned face, unsigned level, unsigned zslice,
+		  enum pipe_transfer_usage usage,
+		  unsigned x, unsigned y, unsigned w, unsigned h)
+{
+	struct nv40_miptree *mt = (struct nv40_miptree *)pt;
+	struct nv40_transfer *tx;
+	struct pipe_texture tx_tex_template, *tx_tex;
+
+	tx = CALLOC_STRUCT(nv40_transfer);
+	if (!tx)
+		return NULL;
+
+	pipe_texture_reference(&tx->base.texture, pt);
+	tx->base.format = pt->format;
+	tx->base.x = x;
+	tx->base.y = y;
+	tx->base.width = w;
+	tx->base.height = h;
+	tx->base.block = pt->block;
+	tx->base.nblocksx = pt->nblocksx[level];
+	tx->base.nblocksy = pt->nblocksy[level];
+	tx->base.stride = mt->level[level].pitch;
+	tx->base.usage = usage;
+	tx->base.face = face;
+	tx->base.level = level;
+	tx->base.zslice = zslice;
+
+	/* Direct access to texture */
+	if ((pt->tex_usage & PIPE_TEXTURE_USAGE_DYNAMIC ||
+	     debug_get_bool_option("NOUVEAU_NO_TRANSFER", TRUE/*XXX:FALSE*/)) &&
+	    pt->tex_usage & NOUVEAU_TEXTURE_USAGE_LINEAR)
+	{
+		tx->direct = true;
+		tx->surface = pscreen->get_tex_surface(pscreen, pt,
+	                                               face, level, zslice,
+	                                               nv40_usage_tx_to_buf(usage));
+		return &tx->base;
+	}
+
+	tx->direct = false;
+
+	nv40_compatible_transfer_tex(pt, level, &tx_tex_template);
+
+	tx_tex = pscreen->texture_create(pscreen, &tx_tex_template);
+	if (!tx_tex)
+	{
+		FREE(tx);
+		return NULL;
+	}
+
+	tx->surface = pscreen->get_tex_surface(pscreen, tx_tex,
+	                                       0, 0, 0,
+	                                       nv40_usage_tx_to_buf(usage));
+
+	pipe_texture_reference(&tx_tex, NULL);
+
+	if (!tx->surface)
+	{
+		pipe_surface_reference(&tx->surface, NULL);
+		FREE(tx);
+		return NULL;
+	}
+
+	if (usage != PIPE_TRANSFER_WRITE) {
+		struct nv40_screen *nvscreen = nv40_screen(pscreen);
+		struct pipe_surface *src;
+
+		src = pscreen->get_tex_surface(pscreen, pt,
+	                                       face, level, zslice,
+	                                       PIPE_BUFFER_USAGE_GPU_READ);
+
+		/* TODO: Check if SIFM can deal with x,y,w,h when swizzling */
+		/* TODO: Check if SIFM can un-swizzle */
+		nvscreen->eng2d->copy(nvscreen->eng2d,
+		                      tx->surface, 0, 0,
+		                      src, 0, 0,
+		                      src->width, src->height);
+
+		pipe_surface_reference(&src, NULL);
+	}
+
+	return &tx->base;
+}
+
+static void
+nv40_transfer_del(struct pipe_screen *pscreen, struct pipe_transfer *ptx)
+{
+	struct nv40_transfer *tx = (struct nv40_transfer *)ptx;
+
+	if (!tx->direct && ptx->usage != PIPE_TRANSFER_READ) {
+		struct pipe_screen *pscreen = ptx->texture->screen;
+		struct nv40_screen *nvscreen = nv40_screen(pscreen);
+		struct pipe_surface *dst;
+
+		dst = pscreen->get_tex_surface(pscreen, ptx->texture,
+	                                       ptx->face, ptx->level, ptx->zslice,
+	                                       PIPE_BUFFER_USAGE_GPU_WRITE);
+
+		/* TODO: Check if SIFM can deal with x,y,w,h when swizzling */
+		nvscreen->eng2d->copy(nvscreen->eng2d,
+		                      dst, 0, 0,
+		                      tx->surface, 0, 0,
+		                      dst->width, dst->height);
+
+		pipe_surface_reference(&dst, NULL);
+	}
+
+	pipe_surface_reference(&tx->surface, NULL);
+	pipe_texture_reference(&ptx->texture, NULL);
+	FREE(ptx);
+}
+
+static void *
+nv40_transfer_map(struct pipe_screen *pscreen, struct pipe_transfer *ptx)
+{
+	struct nv40_transfer *tx = (struct nv40_transfer *)ptx;
+	struct nv04_surface *ns = (struct nv04_surface *)tx->surface;
+	struct nv40_miptree *mt = (struct nv40_miptree *)tx->surface->texture;
+	void *map = pipe_buffer_map(pscreen, mt->buffer,
+	                            nv40_usage_tx_to_buf(ptx->usage));
+
+	return map + ns->base.offset +
+	       ptx->y * ns->pitch + ptx->x * ptx->block.size;
+}
+
+static void
+nv40_transfer_unmap(struct pipe_screen *pscreen, struct pipe_transfer *ptx)
+{
+	struct nv40_transfer *tx = (struct nv40_transfer *)ptx;
+	struct nv40_miptree *mt = (struct nv40_miptree *)tx->surface->texture;
+
+	pipe_buffer_unmap(pscreen, mt->buffer);
+}
+
+void
+nv40_screen_init_transfer_functions(struct pipe_screen *pscreen)
+{
+	pscreen->get_tex_transfer = nv40_transfer_new;
+	pscreen->tex_transfer_destroy = nv40_transfer_del;
+	pscreen->transfer_map = nv40_transfer_map;
+	pscreen->transfer_unmap = nv40_transfer_unmap;
+}
diff --git a/src/gallium/drivers/nv40/nv40_vbo.c b/src/gallium/drivers/nv40/nv40_vbo.c
index 8f1834628f..f3518b2e4f 100644
--- a/src/gallium/drivers/nv40/nv40_vbo.c
+++ b/src/gallium/drivers/nv40/nv40_vbo.c
@@ -537,10 +537,13 @@ nv40_vbo_validate(struct nv40_context *nv40)
 	so_data  (vtxbuf, 0);
 
 	so_ref(vtxbuf, &nv40->state.hw[NV40_STATE_VTXBUF]);
+	so_ref(NULL, &vtxbuf);
 	nv40->state.dirty |= (1ULL << NV40_STATE_VTXBUF);
 	so_ref(vtxfmt, &nv40->state.hw[NV40_STATE_VTXFMT]);
+	so_ref(NULL, &vtxfmt);
 	nv40->state.dirty |= (1ULL << NV40_STATE_VTXFMT);
 	so_ref(sattr, &nv40->state.hw[NV40_STATE_VTXATTR]);
+	so_ref(NULL, &sattr);
 	nv40->state.dirty |= (1ULL << NV40_STATE_VTXATTR);
 	return FALSE;
 }
diff --git a/src/gallium/drivers/nv40/nv40_vertprog.c b/src/gallium/drivers/nv40/nv40_vertprog.c
index 0862386638..7df9a4d326 100644
--- a/src/gallium/drivers/nv40/nv40_vertprog.c
+++ b/src/gallium/drivers/nv40/nv40_vertprog.c
@@ -916,6 +916,7 @@ check_gpu_resources:
 		so_method(so, curie,  NV40TCL_CLIP_PLANE_ENABLE, 1);
 		so_data  (so, vp->clip_ctrl);
 		so_ref(so, &vp->so);
+		so_ref(NULL, &so);
 
 		upload_code = TRUE;
 	}