diff options
Diffstat (limited to 'src/gallium/drivers/nv50')
-rw-r--r-- | src/gallium/drivers/nv50/nv50_context.c | 11 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_context.h | 16 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_program.c | 58 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_query.c | 40 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_screen.c | 26 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_screen.h | 2 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_state.c | 68 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_state_validate.c | 3 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_tex.c | 3 | ||||
-rw-r--r-- | src/gallium/drivers/nv50/nv50_vbo.c | 473 |
10 files changed, 586 insertions, 114 deletions
diff --git a/src/gallium/drivers/nv50/nv50_context.c b/src/gallium/drivers/nv50/nv50_context.c index 22a8498c69..5c705ccc8f 100644 --- a/src/gallium/drivers/nv50/nv50_context.c +++ b/src/gallium/drivers/nv50/nv50_context.c @@ -34,6 +34,11 @@ nv50_flush(struct pipe_context *pipe, unsigned flags, struct nv50_context *nv50 = nv50_context(pipe); struct nouveau_channel *chan = nv50->screen->base.channel; + if (flags & PIPE_FLUSH_TEXTURE_CACHE) { + BEGIN_RING(chan, nv50->screen->tesla, 0x1338, 1); + OUT_RING (chan, 0x20); + } + if (flags & PIPE_FLUSH_FRAME) FIRE_RING(chan); } @@ -81,6 +86,10 @@ nv50_destroy(struct pipe_context *pipe) so_ref(NULL, &nv50->state.vtxattr); draw_destroy(nv50->draw); + + if (nv50->screen->cur_ctx == nv50) + nv50->screen->cur_ctx = NULL; + FREE(nv50); } @@ -104,7 +113,9 @@ nv50_create(struct pipe_screen *pscreen, unsigned pctx_id) nv50->pipe.destroy = nv50_destroy; nv50->pipe.draw_arrays = nv50_draw_arrays; + nv50->pipe.draw_arrays_instanced = nv50_draw_arrays_instanced; nv50->pipe.draw_elements = nv50_draw_elements; + nv50->pipe.draw_elements_instanced = nv50_draw_elements_instanced; nv50->pipe.clear = nv50_clear; nv50->pipe.flush = nv50_flush; diff --git a/src/gallium/drivers/nv50/nv50_context.h b/src/gallium/drivers/nv50/nv50_context.h index d024be6ea9..bebcd95054 100644 --- a/src/gallium/drivers/nv50/nv50_context.h +++ b/src/gallium/drivers/nv50/nv50_context.h @@ -29,9 +29,7 @@ #define NV50_CB_PVP 1 #define NV50_CB_PFP 2 #define NV50_CB_PGP 3 -#define NV50_CB_TIC 4 -#define NV50_CB_TSC 5 -#define NV50_CB_PUPLOAD 6 +#define NV50_CB_AUX 4 #define NV50_NEW_BLEND (1 << 0) #define NV50_NEW_ZSA (1 << 1) @@ -137,6 +135,7 @@ struct nv50_state { struct nouveau_stateobj *vtxfmt; struct nouveau_stateobj *vtxbuf; struct nouveau_stateobj *vtxattr; + struct nouveau_stateobj *instbuf; unsigned vtxelt_nr; }; @@ -198,11 +197,22 @@ extern struct draw_stage *nv50_draw_render_stage(struct nv50_context *nv50); /* nv50_vbo.c */ extern void nv50_draw_arrays(struct pipe_context *, unsigned mode, unsigned start, unsigned count); +extern void nv50_draw_arrays_instanced(struct pipe_context *, unsigned mode, + unsigned start, unsigned count, + unsigned startInstance, + unsigned instanceCount); extern void nv50_draw_elements(struct pipe_context *pipe, struct pipe_buffer *indexBuffer, unsigned indexSize, unsigned mode, unsigned start, unsigned count); +extern void nv50_draw_elements_instanced(struct pipe_context *pipe, + struct pipe_buffer *indexBuffer, + unsigned indexSize, + unsigned mode, unsigned start, + unsigned count, + unsigned startInstance, + unsigned instanceCount); extern void nv50_vbo_validate(struct nv50_context *nv50); /* nv50_clear.c */ diff --git a/src/gallium/drivers/nv50/nv50_program.c b/src/gallium/drivers/nv50/nv50_program.c index acd695b39b..e74a526c62 100644 --- a/src/gallium/drivers/nv50/nv50_program.c +++ b/src/gallium/drivers/nv50/nv50_program.c @@ -95,6 +95,8 @@ struct nv50_reg { int vtx; /* vertex index, for GP inputs (TGSI Dimension.Index) */ int indirect[2]; /* index into pc->addr, or -1 */ + + ubyte buf_index; /* c{0 .. 15}[] or g{0 .. 15}[] */ }; #define NV50_MOD_NEG 1 @@ -188,6 +190,7 @@ ctor_reg(struct nv50_reg *reg, unsigned type, int index, int hw) reg->vtx = -1; reg->acc = 0; reg->indirect[0] = reg->indirect[1] = -1; + reg->buf_index = (type == P_CONST) ? 1 : 0; } static INLINE unsigned @@ -474,6 +477,12 @@ is_join(struct nv50_program_exec *e) return FALSE; } +static INLINE boolean +is_control_flow(struct nv50_program_exec *e) +{ + return (e->inst[0] & 2); +} + static INLINE void set_pred(struct nv50_pc *pc, unsigned pred, unsigned idx, struct nv50_program_exec *e) @@ -631,7 +640,7 @@ set_data(struct nv50_pc *pc, struct nv50_reg *src, unsigned m, unsigned s, set_addr(e, pc->addr[src->indirect[0]]); } - e->inst[1] |= (((src->type == P_IMMD) ? 0 : 1) << 22); + e->inst[1] |= (src->buf_index << 22); } /* Never apply nv50_reg::mod in emit_mov, or carefully check the code !!! */ @@ -3163,7 +3172,9 @@ nv50_program_tx_insn(struct nv50_pc *pc, if (!is_long(pc->p->exec_tail)) convert_to_long(pc, pc->p->exec_tail); else - if (is_immd(pc->p->exec_tail) || is_join(pc->p->exec_tail)) + if (is_immd(pc->p->exec_tail) || + is_join(pc->p->exec_tail) || + is_control_flow(pc->p->exec_tail)) emit_nop(pc); pc->p->exec_tail->inst[1] |= 1; /* set exit bit */ @@ -3272,7 +3283,7 @@ prep_inspect_insn(struct nv50_pc *pc, const struct tgsi_full_instruction *insn) static unsigned nv50_revdep_reorder(unsigned m[4], unsigned rdep[4]) { - unsigned i, c, x, unsafe; + unsigned i, c, x, unsafe = 0; for (c = 0; c < 4; c++) m[c] = c; @@ -3483,6 +3494,19 @@ load_frontfacing(struct nv50_pc *pc, struct nv50_reg *sv) } static void +load_instance_id(struct nv50_pc *pc, unsigned index) +{ + struct nv50_reg reg, mem; + + ctor_reg(®, P_TEMP, -1, -1); + ctor_reg(&mem, P_CONST, -1, 24); /* startInstance */ + mem.buf_index = 2; + + emit_add_b32(pc, ®, &pc->sysval[index], &mem); + pc->sysval[index] = reg; +} + +static void copy_semantic_info(struct nv50_program *p) { unsigned i, id; @@ -3668,8 +3692,10 @@ nv50_program_tx_prep(struct nv50_pc *pc) } if (p->cfg.regs[0] & (1 << 0)) pc->sysval[vertex_id].hw = rid++; - if (p->cfg.regs[0] & (1 << 4)) + if (p->cfg.regs[0] & (1 << 4)) { pc->sysval[instance_id].hw = rid++; + load_instance_id(pc, instance_id); + } } for (i = 0, rid = 0; i < pc->result_nr; ++i) { @@ -3702,13 +3728,21 @@ nv50_program_tx_prep(struct nv50_pc *pc) copy_semantic_info(p); } else if (p->type == PIPE_SHADER_FRAGMENT) { - int rid, aid, base; + int rid, aid; unsigned n = 0, m = pc->attr_nr - flat_nr; pc->allow32 = TRUE; - base = (TGSI_SEMANTIC_POSITION == - p->info.input_semantic_name[0]) ? 0 : 1; + /* do we read FragCoord ? */ + if (pc->attr_nr && + p->info.input_semantic_name[0] == TGSI_SEMANTIC_POSITION) { + /* select FCRD components we want accessible */ + for (c = 0; c < 4; ++c) + if (pc->attr[c].acc) + p->cfg.regs[1] |= 1 << (24 + c); + aid = 0; + } else /* offset by 1 if FCRD.w is needed for pinterp */ + aid = popcnt4(p->cfg.regs[1] >> 24); /* non-flat interpolants have to be mapped to * the lower hardware IDs, so sort them: @@ -3724,12 +3758,6 @@ nv50_program_tx_prep(struct nv50_pc *pc) } copy_semantic_info(p); - if (!base) /* set w-coordinate mask from perspective interp */ - p->cfg.in[0].mask |= p->cfg.regs[1] >> 24; - - aid = popcnt4( /* if fcrd isn't contained in cfg.io */ - base ? (p->cfg.regs[1] >> 24) : p->cfg.in[0].mask); - for (n = 0; n < pc->attr_nr; ++n) { p->cfg.in[n].hw = rid = aid; i = p->cfg.in[n].id; @@ -3751,9 +3779,6 @@ nv50_program_tx_prep(struct nv50_pc *pc) aid += popcnt4(p->cfg.in[n].mask); } - if (!base) - p->cfg.regs[1] |= p->cfg.in[0].mask << 24; - m = popcnt4(p->cfg.regs[1] >> 24); /* set count of non-position inputs and of non-flat @@ -4641,6 +4666,7 @@ nv50_program_destroy(struct nv50_context *nv50, struct nv50_program *p) nouveau_bo_ref(NULL, &p->bo); + FREE(p->immd); nouveau_resource_free(&p->data[0]); p->translated = 0; diff --git a/src/gallium/drivers/nv50/nv50_query.c b/src/gallium/drivers/nv50/nv50_query.c index 5a4ab3508b..9eba4c9611 100644 --- a/src/gallium/drivers/nv50/nv50_query.c +++ b/src/gallium/drivers/nv50/nv50_query.c @@ -48,7 +48,7 @@ nv50_query_create(struct pipe_context *pipe, unsigned type) assert (q->type == PIPE_QUERY_OCCLUSION_COUNTER); q->type = type; - ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM | NOUVEAU_BO_MAP, 256, + ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 256, 16, &q->bo); if (ret) { FREE(q); @@ -95,11 +95,13 @@ nv50_query_end(struct pipe_context *pipe, struct pipe_query *pq) MARK_RING (chan, 5, 2); /* flush on lack of space or relocs */ BEGIN_RING(chan, tesla, NV50TCL_QUERY_ADDRESS_HIGH, 4); - OUT_RELOCh(chan, q->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR); - OUT_RELOCl(chan, q->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_WR); + OUT_RELOCh(chan, q->bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR); + OUT_RELOCl(chan, q->bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_WR); OUT_RING (chan, 0x00000000); OUT_RING (chan, 0x0100f002); - FIRE_RING (chan); + + BEGIN_RING(chan, tesla, NV50TCL_SAMPLECNT_ENABLE, 1); + OUT_RING (chan, 0); } static boolean @@ -123,6 +125,35 @@ nv50_query_result(struct pipe_context *pipe, struct pipe_query *pq, return q->ready; } +static void +nv50_render_condition(struct pipe_context *pipe, + struct pipe_query *pq, uint mode) +{ + struct nv50_context *nv50 = nv50_context(pipe); + struct nouveau_channel *chan = nv50->screen->base.channel; + struct nouveau_grobj *tesla = nv50->screen->tesla; + struct nv50_query *q; + + if (!pq) { + BEGIN_RING(chan, tesla, NV50TCL_COND_MODE, 1); + OUT_RING (chan, NV50TCL_COND_MODE_ALWAYS); + return; + } + q = nv50_query(pq); + + if (mode == PIPE_RENDER_COND_WAIT || + mode == PIPE_RENDER_COND_BY_REGION_WAIT) { + /* XXX: big fence, FIFO semaphore might be better */ + BEGIN_RING(chan, tesla, 0x0110, 1); + OUT_RING (chan, 0); + } + + BEGIN_RING(chan, tesla, NV50TCL_COND_ADDRESS_HIGH, 3); + OUT_RELOCh(chan, q->bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RD); + OUT_RELOCl(chan, q->bo, 0, NOUVEAU_BO_GART | NOUVEAU_BO_RD); + OUT_RING (chan, NV50TCL_COND_MODE_RES); +} + void nv50_init_query_functions(struct nv50_context *nv50) { @@ -131,4 +162,5 @@ nv50_init_query_functions(struct nv50_context *nv50) nv50->pipe.begin_query = nv50_query_begin; nv50->pipe.end_query = nv50_query_end; nv50->pipe.get_query_result = nv50_query_result; + nv50->pipe.render_condition = nv50_render_condition; } diff --git a/src/gallium/drivers/nv50/nv50_screen.c b/src/gallium/drivers/nv50/nv50_screen.c index 73205652cc..c4465a228c 100644 --- a/src/gallium/drivers/nv50/nv50_screen.c +++ b/src/gallium/drivers/nv50/nv50_screen.c @@ -135,6 +135,16 @@ nv50_screen_get_param(struct pipe_screen *pscreen, int param) return 1; case NOUVEAU_CAP_HW_IDXBUF: return 0; + case PIPE_CAP_INDEP_BLEND_ENABLE: + return 1; + case PIPE_CAP_INDEP_BLEND_FUNC: + return 0; + case PIPE_CAP_TGSI_FS_COORD_ORIGIN_UPPER_LEFT: + case PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_HALF_INTEGER: + return 1; + case PIPE_CAP_TGSI_FS_COORD_ORIGIN_LOWER_LEFT: + case PIPE_CAP_TGSI_FS_COORD_PIXEL_CENTER_INTEGER: + return 0; default: NOUVEAU_ERR("Unknown PIPE_CAP %d\n", param); return 0; @@ -329,7 +339,7 @@ nv50_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) so_ref(NULL, &so); /* Static tesla init */ - so = so_new(44, 90, 22); + so = so_new(47, 95, 24); so_method(so, screen->tesla, NV50TCL_COND_MODE, 1); so_data (so, NV50TCL_COND_MODE_ALWAYS); @@ -372,7 +382,7 @@ nv50_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) } for (i = 0; i < 3; i++) { - ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 0, (128 * 4) * 4, + ret = nouveau_bo_new(dev, NOUVEAU_BO_VRAM, 0, (256 * 4) * 4, &screen->constbuf_parm[i]); if (ret) { nv50_screen_destroy(pscreen); @@ -411,6 +421,18 @@ nv50_screen_create(struct pipe_winsys *ws, struct nouveau_device *dev) so_method(so, screen->tesla, NV50TCL_SET_PROGRAM_CB, 1); so_data (so, 0x00000031 | (NV50_CB_PMISC << 12)); + /* bind auxiliary constbuf to immediate data bo */ + so_method(so, screen->tesla, NV50TCL_CB_DEF_ADDRESS_HIGH, 3); + so_reloc (so, screen->constbuf_misc[0], (128 * 4) * 4, + NOUVEAU_BO_VRAM | NOUVEAU_BO_RD | NOUVEAU_BO_HIGH, 0, 0); + so_reloc (so, screen->constbuf_misc[0], (128 * 4) * 4, + NOUVEAU_BO_VRAM | NOUVEAU_BO_RD | NOUVEAU_BO_LOW, 0, 0); + so_data (so, (NV50_CB_AUX << 16) | 0x00000200); + so_method(so, screen->tesla, NV50TCL_SET_PROGRAM_CB, 1); + so_data (so, 0x00000201 | (NV50_CB_AUX << 12)); + so_method(so, screen->tesla, NV50TCL_SET_PROGRAM_CB, 1); + so_data (so, 0x00000221 | (NV50_CB_AUX << 12)); + so_method(so, screen->tesla, NV50TCL_CB_DEF_ADDRESS_HIGH, 3); so_reloc (so, screen->constbuf_parm[PIPE_SHADER_VERTEX], 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD | NOUVEAU_BO_HIGH, 0, 0); diff --git a/src/gallium/drivers/nv50/nv50_screen.h b/src/gallium/drivers/nv50/nv50_screen.h index 7a155ca9c4..0d786b0f2e 100644 --- a/src/gallium/drivers/nv50/nv50_screen.h +++ b/src/gallium/drivers/nv50/nv50_screen.h @@ -23,6 +23,8 @@ struct nv50_screen { struct nouveau_resource *immd_heap[1]; struct nouveau_resource *parm_heap[PIPE_SHADER_TYPES]; + struct pipe_buffer *strm_vbuf[16]; + struct nouveau_bo *tic; struct nouveau_bo *tsc; diff --git a/src/gallium/drivers/nv50/nv50_state.c b/src/gallium/drivers/nv50/nv50_state.c index 6ab33be663..da8dc933b0 100644 --- a/src/gallium/drivers/nv50/nv50_state.c +++ b/src/gallium/drivers/nv50/nv50_state.c @@ -31,6 +31,23 @@ #include "nouveau/nouveau_stateobj.h" +static INLINE uint32_t +nv50_colormask(unsigned mask) +{ + uint32_t cmask = 0; + + if (mask & PIPE_MASK_R) + cmask |= 0x0001; + if (mask & PIPE_MASK_G) + cmask |= 0x0010; + if (mask & PIPE_MASK_B) + cmask |= 0x0100; + if (mask & PIPE_MASK_A) + cmask |= 0x1000; + + return cmask; +} + static void * nv50_blend_state_create(struct pipe_context *pipe, const struct pipe_blend_state *cso) @@ -38,28 +55,37 @@ nv50_blend_state_create(struct pipe_context *pipe, struct nouveau_stateobj *so = so_new(5, 24, 0); struct nouveau_grobj *tesla = nv50_context(pipe)->screen->tesla; struct nv50_blend_stateobj *bso = CALLOC_STRUCT(nv50_blend_stateobj); - unsigned cmask = 0, i; + unsigned i, blend_enabled = 0; /*XXX ignored: * - dither */ - if (cso->blend_enable == 0) { - so_method(so, tesla, NV50TCL_BLEND_ENABLE(0), 8); + so_method(so, tesla, NV50TCL_BLEND_ENABLE(0), 8); + if (cso->independent_blend_enable) { + for (i = 0; i < 8; ++i) { + so_data(so, cso->rt[i].blend_enable); + if (cso->rt[i].blend_enable) + blend_enabled = 1; + } + } else + if (cso->rt[0].blend_enable) { + blend_enabled = 1; for (i = 0; i < 8; i++) - so_data(so, 0); + so_data(so, 1); } else { - so_method(so, tesla, NV50TCL_BLEND_ENABLE(0), 8); for (i = 0; i < 8; i++) - so_data(so, 1); + so_data(so, 0); + } + if (blend_enabled) { so_method(so, tesla, NV50TCL_BLEND_EQUATION_RGB, 5); - so_data (so, nvgl_blend_eqn(cso->rgb_func)); - so_data (so, 0x4000 | nvgl_blend_func(cso->rgb_src_factor)); - so_data (so, 0x4000 | nvgl_blend_func(cso->rgb_dst_factor)); - so_data (so, nvgl_blend_eqn(cso->alpha_func)); - so_data (so, 0x4000 | nvgl_blend_func(cso->alpha_src_factor)); + so_data (so, nvgl_blend_eqn(cso->rt[0].rgb_func)); + so_data (so, 0x4000 | nvgl_blend_func(cso->rt[0].rgb_src_factor)); + so_data (so, 0x4000 | nvgl_blend_func(cso->rt[0].rgb_dst_factor)); + so_data (so, nvgl_blend_eqn(cso->rt[0].alpha_func)); + so_data (so, 0x4000 | nvgl_blend_func(cso->rt[0].alpha_src_factor)); so_method(so, tesla, NV50TCL_BLEND_FUNC_DST_ALPHA, 1); - so_data (so, 0x4000 | nvgl_blend_func(cso->alpha_dst_factor)); + so_data (so, 0x4000 | nvgl_blend_func(cso->rt[0].alpha_dst_factor)); } if (cso->logicop_enable == 0 ) { @@ -71,17 +97,15 @@ nv50_blend_state_create(struct pipe_context *pipe, so_data (so, nvgl_logicop_func(cso->logicop_func)); } - if (cso->colormask & PIPE_MASK_R) - cmask |= (1 << 0); - if (cso->colormask & PIPE_MASK_G) - cmask |= (1 << 4); - if (cso->colormask & PIPE_MASK_B) - cmask |= (1 << 8); - if (cso->colormask & PIPE_MASK_A) - cmask |= (1 << 12); so_method(so, tesla, NV50TCL_COLOR_MASK(0), 8); - for (i = 0; i < 8; i++) - so_data(so, cmask); + if (cso->independent_blend_enable) + for (i = 0; i < 8; ++i) + so_data(so, nv50_colormask(cso->rt[i].colormask)); + else { + uint32_t cmask = nv50_colormask(cso->rt[0].colormask); + for (i = 0; i < 8; i++) + so_data(so, cmask); + } bso->pipe = *cso; so_ref(so, &bso->so); diff --git a/src/gallium/drivers/nv50/nv50_state_validate.c b/src/gallium/drivers/nv50/nv50_state_validate.c index fcd07b59cd..956da9b304 100644 --- a/src/gallium/drivers/nv50/nv50_state_validate.c +++ b/src/gallium/drivers/nv50/nv50_state_validate.c @@ -274,6 +274,9 @@ nv50_state_flush_notify(struct nouveau_channel *chan) so_emit_reloc_markers(chan, nv50->state.fragprog); so_emit_reloc_markers(chan, nv50->state.vtxbuf); so_emit_reloc_markers(chan, nv50->screen->static_init); + + if (nv50->state.instbuf) + so_emit_reloc_markers(chan, nv50->state.instbuf); } boolean diff --git a/src/gallium/drivers/nv50/nv50_tex.c b/src/gallium/drivers/nv50/nv50_tex.c index 871536dca9..9f1a171303 100644 --- a/src/gallium/drivers/nv50/nv50_tex.c +++ b/src/gallium/drivers/nv50/nv50_tex.c @@ -220,11 +220,8 @@ nv50_tex_validate(struct nv50_context *nv50) return; } - /* not sure if the following really do what I think: */ so_method(so, tesla, 0x1330, 1); /* flush TIC */ so_data (so, 0); - so_method(so, tesla, 0x1338, 1); /* flush texture caches */ - so_data (so, 0x20); so_ref(so, &nv50->state.tic_upload); so_ref(NULL, &so); diff --git a/src/gallium/drivers/nv50/nv50_vbo.c b/src/gallium/drivers/nv50/nv50_vbo.c index 89a94d2fe8..bfb1b34d27 100644 --- a/src/gallium/drivers/nv50/nv50_vbo.c +++ b/src/gallium/drivers/nv50/nv50_vbo.c @@ -40,6 +40,8 @@ nv50_push_elements_u32(struct nv50_context *, uint32_t *, unsigned); static boolean nv50_push_arrays(struct nv50_context *, unsigned, unsigned); +#define NV50_USING_LOATHED_EDGEFLAG(ctx) ((ctx)->vertprog->cfg.edgeflag_in < 16) + static INLINE unsigned nv50_prim(unsigned mode) { @@ -160,6 +162,309 @@ nv50_vbo_vtxelt_to_hw(struct pipe_vertex_element *ve) return (hw_type | hw_size); } +/* For instanced drawing from user buffers, hitting the FIFO repeatedly + * with the same vertex data is probably worse than uploading all data. + */ +static boolean +nv50_upload_vtxbuf(struct nv50_context *nv50, unsigned i) +{ + struct nv50_screen *nscreen = nv50->screen; + struct pipe_screen *pscreen = &nscreen->base.base; + struct pipe_buffer *buf = nscreen->strm_vbuf[i]; + struct pipe_vertex_buffer *vb = &nv50->vtxbuf[i]; + uint8_t *src; + unsigned size = align(vb->buffer->size, 4096); + + if (buf && buf->size < size) + pipe_buffer_reference(&nscreen->strm_vbuf[i], NULL); + + if (!nscreen->strm_vbuf[i]) { + nscreen->strm_vbuf[i] = pipe_buffer_create( + pscreen, 0, PIPE_BUFFER_USAGE_VERTEX, size); + buf = nscreen->strm_vbuf[i]; + } + + src = pipe_buffer_map(pscreen, vb->buffer, PIPE_BUFFER_USAGE_CPU_READ); + if (!src) + return FALSE; + src += vb->buffer_offset; + + size = (vb->max_index + 1) * vb->stride + 16; /* + 16 is for stride 0 */ + if (vb->buffer_offset + size > vb->buffer->size) + size = vb->buffer->size - vb->buffer_offset; + + pipe_buffer_write(pscreen, buf, vb->buffer_offset, size, src); + pipe_buffer_unmap(pscreen, vb->buffer); + + vb->buffer = buf; /* don't pipe_reference, this is a private copy */ + return TRUE; +} + +static void +nv50_upload_user_vbufs(struct nv50_context *nv50) +{ + unsigned i; + + if (nv50->vbo_fifo) + nv50->dirty |= NV50_NEW_ARRAYS; + if (!(nv50->dirty & NV50_NEW_ARRAYS)) + return; + + for (i = 0; i < nv50->vtxbuf_nr; ++i) { + if (nv50->vtxbuf[i].buffer->usage & PIPE_BUFFER_USAGE_VERTEX) + continue; + nv50_upload_vtxbuf(nv50, i); + } +} + +static void +nv50_set_static_vtxattr(struct nv50_context *nv50, unsigned i, void *data) +{ + struct nouveau_grobj *tesla = nv50->screen->tesla; + struct nouveau_channel *chan = tesla->channel; + float v[4]; + + util_format_read_4f(nv50->vtxelt[i].src_format, + v, 0, data, 0, 0, 0, 1, 1); + + switch (nv50->vtxelt[i].nr_components) { + case 4: + BEGIN_RING(chan, tesla, NV50TCL_VTX_ATTR_4F_X(i), 4); + OUT_RINGf (chan, v[0]); + OUT_RINGf (chan, v[1]); + OUT_RINGf (chan, v[2]); + OUT_RINGf (chan, v[3]); + break; + case 3: + BEGIN_RING(chan, tesla, NV50TCL_VTX_ATTR_3F_X(i), 3); + OUT_RINGf (chan, v[0]); + OUT_RINGf (chan, v[1]); + OUT_RINGf (chan, v[2]); + break; + case 2: + BEGIN_RING(chan, tesla, NV50TCL_VTX_ATTR_2F_X(i), 2); + OUT_RINGf (chan, v[0]); + OUT_RINGf (chan, v[1]); + break; + case 1: + BEGIN_RING(chan, tesla, NV50TCL_VTX_ATTR_1F(i), 1); + OUT_RINGf (chan, v[0]); + break; + default: + assert(0); + break; + } +} + +static unsigned +init_per_instance_arrays_immd(struct nv50_context *nv50, + unsigned startInstance, + unsigned pos[16], unsigned step[16]) +{ + struct nouveau_bo *bo; + unsigned i, b, count = 0; + + for (i = 0; i < nv50->vtxelt_nr; ++i) { + if (!nv50->vtxelt[i].instance_divisor) + continue; + ++count; + b = nv50->vtxelt[i].vertex_buffer_index; + + pos[i] = nv50->vtxelt[i].src_offset + + nv50->vtxbuf[b].buffer_offset + + startInstance * nv50->vtxbuf[b].stride; + step[i] = startInstance % nv50->vtxelt[i].instance_divisor; + + bo = nouveau_bo(nv50->vtxbuf[b].buffer); + if (!bo->map) + nouveau_bo_map(bo, NOUVEAU_BO_RD); + + nv50_set_static_vtxattr(nv50, i, (uint8_t *)bo->map + pos[i]); + } + + return count; +} + +static unsigned +init_per_instance_arrays(struct nv50_context *nv50, + unsigned startInstance, + unsigned pos[16], unsigned step[16]) +{ + struct nouveau_grobj *tesla = nv50->screen->tesla; + struct nouveau_channel *chan = tesla->channel; + struct nouveau_bo *bo; + struct nouveau_stateobj *so; + unsigned i, b, count = 0; + const uint32_t rl = NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_RD; + + if (nv50->vbo_fifo) + return init_per_instance_arrays_immd(nv50, startInstance, + pos, step); + + so = so_new(nv50->vtxelt_nr, nv50->vtxelt_nr * 2, nv50->vtxelt_nr * 2); + + for (i = 0; i < nv50->vtxelt_nr; ++i) { + if (!nv50->vtxelt[i].instance_divisor) + continue; + ++count; + b = nv50->vtxelt[i].vertex_buffer_index; + + pos[i] = nv50->vtxelt[i].src_offset + + nv50->vtxbuf[b].buffer_offset + + startInstance * nv50->vtxbuf[b].stride; + + if (!startInstance) { + step[i] = 0; + continue; + } + step[i] = startInstance % nv50->vtxelt[i].instance_divisor; + + bo = nouveau_bo(nv50->vtxbuf[b].buffer); + + so_method(so, tesla, NV50TCL_VERTEX_ARRAY_START_HIGH(i), 2); + so_reloc (so, bo, pos[i], rl | NOUVEAU_BO_HIGH, 0, 0); + so_reloc (so, bo, pos[i], rl | NOUVEAU_BO_LOW, 0, 0); + } + + if (count && startInstance) { + so_ref (so, &nv50->state.instbuf); /* for flush notify */ + so_emit(chan, nv50->state.instbuf); + } + so_ref (NULL, &so); + + return count; +} + +static void +step_per_instance_arrays_immd(struct nv50_context *nv50, + unsigned pos[16], unsigned step[16]) +{ + struct nouveau_bo *bo; + unsigned i, b; + + for (i = 0; i < nv50->vtxelt_nr; ++i) { + if (!nv50->vtxelt[i].instance_divisor) + continue; + if (++step[i] != nv50->vtxelt[i].instance_divisor) + continue; + b = nv50->vtxelt[i].vertex_buffer_index; + bo = nouveau_bo(nv50->vtxbuf[b].buffer); + + step[i] = 0; + pos[i] += nv50->vtxbuf[b].stride; + + nv50_set_static_vtxattr(nv50, i, (uint8_t *)bo->map + pos[i]); + } +} + +static void +step_per_instance_arrays(struct nv50_context *nv50, + unsigned pos[16], unsigned step[16]) +{ + struct nouveau_grobj *tesla = nv50->screen->tesla; + struct nouveau_channel *chan = tesla->channel; + struct nouveau_bo *bo; + struct nouveau_stateobj *so; + unsigned i, b; + const uint32_t rl = NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_RD; + + if (nv50->vbo_fifo) { + step_per_instance_arrays_immd(nv50, pos, step); + return; + } + + so = so_new(nv50->vtxelt_nr, nv50->vtxelt_nr * 2, nv50->vtxelt_nr * 2); + + for (i = 0; i < nv50->vtxelt_nr; ++i) { + if (!nv50->vtxelt[i].instance_divisor) + continue; + b = nv50->vtxelt[i].vertex_buffer_index; + + if (++step[i] == nv50->vtxelt[i].instance_divisor) { + step[i] = 0; + pos[i] += nv50->vtxbuf[b].stride; + } + + bo = nouveau_bo(nv50->vtxbuf[b].buffer); + + so_method(so, tesla, NV50TCL_VERTEX_ARRAY_START_HIGH(i), 2); + so_reloc (so, bo, pos[i], rl | NOUVEAU_BO_HIGH, 0, 0); + so_reloc (so, bo, pos[i], rl | NOUVEAU_BO_LOW, 0, 0); + } + + so_ref (so, &nv50->state.instbuf); /* for flush notify */ + so_ref (NULL, &so); + + so_emit(chan, nv50->state.instbuf); +} + +static INLINE void +nv50_unmap_vbufs(struct nv50_context *nv50) +{ + unsigned i; + + for (i = 0; i < nv50->vtxbuf_nr; ++i) + if (nouveau_bo(nv50->vtxbuf[i].buffer)->map) + nouveau_bo_unmap(nouveau_bo(nv50->vtxbuf[i].buffer)); +} + +void +nv50_draw_arrays_instanced(struct pipe_context *pipe, + unsigned mode, unsigned start, unsigned count, + unsigned startInstance, unsigned instanceCount) +{ + struct nv50_context *nv50 = nv50_context(pipe); + struct nouveau_channel *chan = nv50->screen->tesla->channel; + struct nouveau_grobj *tesla = nv50->screen->tesla; + unsigned i, nz_divisors; + unsigned step[16], pos[16]; + + if (!NV50_USING_LOATHED_EDGEFLAG(nv50)) + nv50_upload_user_vbufs(nv50); + + nv50_state_validate(nv50); + + nz_divisors = init_per_instance_arrays(nv50, startInstance, pos, step); + + BEGIN_RING(chan, tesla, NV50TCL_CB_ADDR, 2); + OUT_RING (chan, NV50_CB_AUX | (24 << 8)); + OUT_RING (chan, startInstance); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BEGIN, 1); + OUT_RING (chan, nv50_prim(mode)); + + if (nv50->vbo_fifo) + nv50_push_arrays(nv50, start, count); + else { + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BUFFER_FIRST, 2); + OUT_RING (chan, start); + OUT_RING (chan, count); + } + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); + OUT_RING (chan, 0); + + for (i = 1; i < instanceCount; i++) { + if (nz_divisors) /* any non-zero array divisors ? */ + step_per_instance_arrays(nv50, pos, step); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BEGIN, 1); + OUT_RING (chan, nv50_prim(mode) | (1 << 28)); + + if (nv50->vbo_fifo) + nv50_push_arrays(nv50, start, count); + else { + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BUFFER_FIRST, 2); + OUT_RING (chan, start); + OUT_RING (chan, count); + } + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); + OUT_RING (chan, 0); + } + nv50_unmap_vbufs(nv50); + + so_ref(NULL, &nv50->state.instbuf); +} + void nv50_draw_arrays(struct pipe_context *pipe, unsigned mode, unsigned start, unsigned count) @@ -190,6 +495,8 @@ nv50_draw_arrays(struct pipe_context *pipe, unsigned mode, unsigned start, BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); OUT_RING (chan, 0); + nv50_unmap_vbufs(nv50); + /* XXX: not sure what to do if ret != TRUE: flush and retry? */ assert(ret); @@ -218,7 +525,7 @@ nv50_draw_elements_inline_u08(struct nv50_context *nv50, uint8_t *map, unsigned nr = count > 2046 ? 2046 : count; int i; - BEGIN_RING(chan, tesla, NV50TCL_VB_ELEMENT_U16 | 0x40000000, nr >> 1); + BEGIN_RING_NI(chan, tesla, NV50TCL_VB_ELEMENT_U16, nr >> 1); for (i = 0; i < nr; i += 2) OUT_RING (chan, (map[i + 1] << 16) | map[i]); @@ -251,7 +558,7 @@ nv50_draw_elements_inline_u16(struct nv50_context *nv50, uint16_t *map, unsigned nr = count > 2046 ? 2046 : count; int i; - BEGIN_RING(chan, tesla, NV50TCL_VB_ELEMENT_U16 | 0x40000000, nr >> 1); + BEGIN_RING_NI(chan, tesla, NV50TCL_VB_ELEMENT_U16, nr >> 1); for (i = 0; i < nr; i += 2) OUT_RING (chan, (map[i + 1] << 16) | map[i]); @@ -276,7 +583,7 @@ nv50_draw_elements_inline_u32(struct nv50_context *nv50, uint32_t *map, while (count) { unsigned nr = count > 2047 ? 2047 : count; - BEGIN_RING(chan, tesla, NV50TCL_VB_ELEMENT_U32 | 0x40000000, nr); + BEGIN_RING_NI(chan, tesla, NV50TCL_VB_ELEMENT_U32, nr); OUT_RINGp (chan, map, nr); count -= nr; @@ -285,6 +592,77 @@ nv50_draw_elements_inline_u32(struct nv50_context *nv50, uint32_t *map, return TRUE; } +static INLINE void +nv50_draw_elements_inline(struct nv50_context *nv50, + void *map, unsigned indexSize, + unsigned start, unsigned count) +{ + switch (indexSize) { + case 1: + nv50_draw_elements_inline_u08(nv50, map, start, count); + break; + case 2: + nv50_draw_elements_inline_u16(nv50, map, start, count); + break; + case 4: + nv50_draw_elements_inline_u32(nv50, map, start, count); + break; + } +} + +void +nv50_draw_elements_instanced(struct pipe_context *pipe, + struct pipe_buffer *indexBuffer, + unsigned indexSize, + unsigned mode, unsigned start, unsigned count, + unsigned startInstance, unsigned instanceCount) +{ + struct nv50_context *nv50 = nv50_context(pipe); + struct nouveau_grobj *tesla = nv50->screen->tesla; + struct nouveau_channel *chan = tesla->channel; + struct pipe_screen *pscreen = pipe->screen; + void *map; + unsigned i, nz_divisors; + unsigned step[16], pos[16]; + + map = pipe_buffer_map(pscreen, indexBuffer, PIPE_BUFFER_USAGE_CPU_READ); + + if (!NV50_USING_LOATHED_EDGEFLAG(nv50)) + nv50_upload_user_vbufs(nv50); + + nv50_state_validate(nv50); + + nz_divisors = init_per_instance_arrays(nv50, startInstance, pos, step); + + BEGIN_RING(chan, tesla, NV50TCL_CB_ADDR, 2); + OUT_RING (chan, NV50_CB_AUX | (24 << 8)); + OUT_RING (chan, startInstance); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BEGIN, 1); + OUT_RING (chan, nv50_prim(mode)); + + nv50_draw_elements_inline(nv50, map, indexSize, start, count); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); + OUT_RING (chan, 0); + + for (i = 1; i < instanceCount; ++i) { + if (nz_divisors) /* any non-zero array divisors ? */ + step_per_instance_arrays(nv50, pos, step); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BEGIN, 1); + OUT_RING (chan, nv50_prim(mode) | (1 << 28)); + + nv50_draw_elements_inline(nv50, map, indexSize, start, count); + + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); + OUT_RING (chan, 0); + } + nv50_unmap_vbufs(nv50); + + so_ref(NULL, &nv50->state.instbuf); +} + void nv50_draw_elements(struct pipe_context *pipe, struct pipe_buffer *indexBuffer, unsigned indexSize, @@ -295,7 +673,6 @@ nv50_draw_elements(struct pipe_context *pipe, struct nouveau_grobj *tesla = nv50->screen->tesla; struct pipe_screen *pscreen = pipe->screen; void *map; - boolean ret; map = pipe_buffer_map(pscreen, indexBuffer, PIPE_BUFFER_USAGE_CPU_READ); @@ -308,29 +685,15 @@ nv50_draw_elements(struct pipe_context *pipe, BEGIN_RING(chan, tesla, NV50TCL_VERTEX_BEGIN, 1); OUT_RING (chan, nv50_prim(mode)); - switch (indexSize) { - case 1: - ret = nv50_draw_elements_inline_u08(nv50, map, start, count); - break; - case 2: - ret = nv50_draw_elements_inline_u16(nv50, map, start, count); - break; - case 4: - ret = nv50_draw_elements_inline_u32(nv50, map, start, count); - break; - default: - assert(0); - ret = FALSE; - break; - } + + nv50_draw_elements_inline(nv50, map, indexSize, start, count); + BEGIN_RING(chan, tesla, NV50TCL_VERTEX_END, 1); OUT_RING (chan, 0); + nv50_unmap_vbufs(nv50); + pipe_buffer_unmap(pscreen, indexBuffer); - - /* XXX: what to do if ret != TRUE? Flush and retry? - */ - assert(ret); } static INLINE boolean @@ -343,23 +706,16 @@ nv50_vbo_static_attrib(struct nv50_context *nv50, unsigned attrib, struct nouveau_stateobj *so; struct nouveau_grobj *tesla = nv50->screen->tesla; struct nouveau_bo *bo = nouveau_bo(vb->buffer); - float *v; + float v[4]; int ret; - enum pipe_format pf = ve->src_format; - const struct util_format_description *desc; - - desc = util_format_description(pf); - assert(desc); - - if ((desc->channel[0].type != UTIL_FORMAT_TYPE_FLOAT) || - util_format_get_component_bits(pf, UTIL_FORMAT_COLORSPACE_RGB, 0) != 32) - return FALSE; ret = nouveau_bo_map(bo, NOUVEAU_BO_RD); if (ret) return FALSE; - v = (float *)(bo->map + (vb->buffer_offset + ve->src_offset)); + util_format_read_4f(ve->src_format, v, 0, (uint8_t *)bo->map + + (vb->buffer_offset + ve->src_offset), 0, + 0, 0, 1, 1); so = *pso; if (!so) *pso = so = so_new(nv50->vtxelt_nr, nv50->vtxelt_nr * 4, 0); @@ -417,7 +773,7 @@ nv50_vbo_validate(struct nv50_context *nv50) !(nv50->vtxbuf[i].buffer->usage & PIPE_BUFFER_USAGE_VERTEX)) nv50->vbo_fifo = 0xffff; - if (nv50->vertprog->cfg.edgeflag_in < 16) + if (NV50_USING_LOATHED_EDGEFLAG(nv50)) nv50->vbo_fifo = 0xffff; /* vertprog can't set edgeflag */ n_ve = MAX2(nv50->vtxelt_nr, nv50->state.vtxelt_nr); @@ -445,17 +801,20 @@ nv50_vbo_validate(struct nv50_context *nv50) nv50->vbo_fifo &= ~(1 << i); continue; } - so_data(vtxfmt, hw | i); if (nv50->vbo_fifo) { + so_data (vtxfmt, hw | + (ve->instance_divisor ? (1 << 4) : i)); so_method(vtxbuf, tesla, NV50TCL_VERTEX_ARRAY_FORMAT(i), 1); so_data (vtxbuf, 0); continue; } + so_data(vtxfmt, hw | i); so_method(vtxbuf, tesla, NV50TCL_VERTEX_ARRAY_FORMAT(i), 3); - so_data (vtxbuf, 0x20000000 | vb->stride); + so_data (vtxbuf, 0x20000000 | + (ve->instance_divisor ? 0 : vb->stride)); so_reloc (vtxbuf, bo, vb->buffer_offset + ve->src_offset, NOUVEAU_BO_VRAM | NOUVEAU_BO_GART | NOUVEAU_BO_RD | NOUVEAU_BO_HIGH, 0, 0); @@ -493,7 +852,7 @@ typedef void (*pfn_push)(struct nouveau_channel *, void *); struct nv50_vbo_emitctx { pfn_push push[16]; - void *map[16]; + uint8_t *map[16]; unsigned stride[16]; unsigned nr_ve; unsigned vtx_dwords; @@ -531,19 +890,18 @@ nv50_map_vbufs(struct nv50_context *nv50) for (i = 0; i < nv50->vtxbuf_nr; ++i) { struct pipe_vertex_buffer *vb = &nv50->vtxbuf[i]; - unsigned size, delta; + unsigned size = vb->stride * (vb->max_index + 1) + 16; if (nouveau_bo(vb->buffer)->map) continue; - size = vb->stride * (vb->max_index + 1); - delta = vb->buffer_offset; - + size = vb->stride * (vb->max_index + 1) + 16; + size = MIN2(size, vb->buffer->size); if (!size) - size = vb->buffer->size - vb->buffer_offset; + size = vb->buffer->size; if (nouveau_bo_map_range(nouveau_bo(vb->buffer), - delta, size, NOUVEAU_BO_RD)) + 0, size, NOUVEAU_BO_RD)) break; } @@ -554,16 +912,6 @@ nv50_map_vbufs(struct nv50_context *nv50) return FALSE; } -static INLINE void -nv50_unmap_vbufs(struct nv50_context *nv50) -{ - unsigned i; - - for (i = 0; i < nv50->vtxbuf_nr; ++i) - if (nouveau_bo(nv50->vtxbuf[i].buffer)->map) - nouveau_bo_unmap(nouveau_bo(nv50->vtxbuf[i].buffer)); -} - static void emit_b32_1(struct nouveau_channel *chan, void *data) { @@ -658,12 +1006,13 @@ emit_prepare(struct nv50_context *nv50, struct nv50_vbo_emitctx *emit, ve = &nv50->vtxelt[i]; vb = &nv50->vtxbuf[ve->vertex_buffer_index]; - if (!(nv50->vbo_fifo & (1 << i))) + if (!(nv50->vbo_fifo & (1 << i)) || ve->instance_divisor) continue; n = emit->nr_ve++; emit->stride[n] = vb->stride; - emit->map[n] = nouveau_bo(vb->buffer)->map + + emit->map[n] = (uint8_t *)nouveau_bo(vb->buffer)->map + + vb->buffer_offset + (start * vb->stride + ve->src_offset); desc = util_format_description(ve->src_format); @@ -753,13 +1102,12 @@ nv50_push_arrays(struct nv50_context *nv50, unsigned start, unsigned count) set_edgeflag(chan, tesla, &emit, 0); /* nr will be 1 */ - BEGIN_RING(chan, tesla, NV50TCL_VERTEX_DATA | 0x40000000, dw); + BEGIN_RING_NI(chan, tesla, NV50TCL_VERTEX_DATA, dw); for (i = 0; i < nr; ++i) emit_vtx_next(chan, &emit); count -= nr; } - nv50_unmap_vbufs(nv50); return TRUE; } @@ -780,13 +1128,12 @@ nv50_push_elements_u32(struct nv50_context *nv50, uint32_t *map, unsigned count) set_edgeflag(chan, tesla, &emit, *map); - BEGIN_RING(chan, tesla, NV50TCL_VERTEX_DATA | 0x40000000, dw); + BEGIN_RING_NI(chan, tesla, NV50TCL_VERTEX_DATA, dw); for (i = 0; i < nr; ++i) emit_vtx(chan, &emit, *map++); count -= nr; } - nv50_unmap_vbufs(nv50); return TRUE; } @@ -807,13 +1154,12 @@ nv50_push_elements_u16(struct nv50_context *nv50, uint16_t *map, unsigned count) set_edgeflag(chan, tesla, &emit, *map); - BEGIN_RING(chan, tesla, NV50TCL_VERTEX_DATA | 0x40000000, dw); + BEGIN_RING_NI(chan, tesla, NV50TCL_VERTEX_DATA, dw); for (i = 0; i < nr; ++i) emit_vtx(chan, &emit, *map++); count -= nr; } - nv50_unmap_vbufs(nv50); return TRUE; } @@ -834,13 +1180,12 @@ nv50_push_elements_u08(struct nv50_context *nv50, uint8_t *map, unsigned count) set_edgeflag(chan, tesla, &emit, *map); - BEGIN_RING(chan, tesla, NV50TCL_VERTEX_DATA | 0x40000000, dw); + BEGIN_RING_NI(chan, tesla, NV50TCL_VERTEX_DATA, dw); for (i = 0; i < nr; ++i) emit_vtx(chan, &emit, *map++); count -= nr; } - nv50_unmap_vbufs(nv50); return TRUE; } |