diff options
author | José Fonseca <jfonseca@vmware.com> | 2010-01-08 15:42:57 +0000 |
---|---|---|
committer | José Fonseca <jfonseca@vmware.com> | 2010-01-08 15:42:57 +0000 |
commit | 080c40ab32b2abd6d8381b4a0cc143d36a1652b2 (patch) | |
tree | e173767ebc5a82d81b9fc086449d915e29348976 /src/gallium/drivers/nv50/nv50_program.c | |
parent | 9cdf6f025b2ed55cfb13dd09f870f01d0c7947d3 (diff) | |
parent | a1de400e8de06a80ab140bb0fa950e990607572d (diff) |
Merge remote branch 'origin/master' into lp-binning
Conflicts:
src/gallium/auxiliary/util/u_surface.c
src/gallium/drivers/llvmpipe/Makefile
src/gallium/drivers/llvmpipe/SConscript
src/gallium/drivers/llvmpipe/lp_bld_arit.c
src/gallium/drivers/llvmpipe/lp_bld_flow.c
src/gallium/drivers/llvmpipe/lp_bld_interp.c
src/gallium/drivers/llvmpipe/lp_clear.c
src/gallium/drivers/llvmpipe/lp_context.c
src/gallium/drivers/llvmpipe/lp_context.h
src/gallium/drivers/llvmpipe/lp_draw_arrays.c
src/gallium/drivers/llvmpipe/lp_jit.c
src/gallium/drivers/llvmpipe/lp_jit.h
src/gallium/drivers/llvmpipe/lp_prim_vbuf.c
src/gallium/drivers/llvmpipe/lp_setup.c
src/gallium/drivers/llvmpipe/lp_setup_point.c
src/gallium/drivers/llvmpipe/lp_state.h
src/gallium/drivers/llvmpipe/lp_state_blend.c
src/gallium/drivers/llvmpipe/lp_state_derived.c
src/gallium/drivers/llvmpipe/lp_state_fs.c
src/gallium/drivers/llvmpipe/lp_state_sampler.c
src/gallium/drivers/llvmpipe/lp_state_surface.c
src/gallium/drivers/llvmpipe/lp_tex_cache.c
src/gallium/drivers/llvmpipe/lp_tex_cache.h
src/gallium/drivers/llvmpipe/lp_tex_sample.h
src/gallium/drivers/llvmpipe/lp_tile_cache.c
Diffstat (limited to 'src/gallium/drivers/nv50/nv50_program.c')
-rw-r--r-- | src/gallium/drivers/nv50/nv50_program.c | 2037 |
1 files changed, 1516 insertions, 521 deletions
diff --git a/src/gallium/drivers/nv50/nv50_program.c b/src/gallium/drivers/nv50/nv50_program.c index 576d075318..53f9f0adf3 100644 --- a/src/gallium/drivers/nv50/nv50_program.c +++ b/src/gallium/drivers/nv50/nv50_program.c @@ -31,9 +31,12 @@ #include "nv50_context.h" -#define NV50_SU_MAX_TEMP 64 +#define NV50_SU_MAX_TEMP 127 +#define NV50_SU_MAX_ADDR 4 //#define NV50_PROGRAM_DUMP +/* $a5 and $a6 always seem to be 0, and using $a7 gives you noise */ + /* ARL - gallium craps itself on progs/vp/arl.txt * * MSB - Like MAD, but MUL+SUB @@ -79,26 +82,44 @@ struct nv50_reg { P_ATTR, P_RESULT, P_CONST, - P_IMMD + P_IMMD, + P_ADDR } type; int index; int hw; - int neg; + int mod; int rhw; /* result hw for FP outputs, or interpolant index */ int acc; /* instruction where this reg is last read (first insn == 1) */ }; -/* arbitrary limits */ -#define MAX_IF_DEPTH 4 -#define MAX_LOOP_DEPTH 4 +#define NV50_MOD_NEG 1 +#define NV50_MOD_ABS 2 +#define NV50_MOD_NEG_ABS (NV50_MOD_NEG | NV50_MOD_ABS) +#define NV50_MOD_SAT 4 +#define NV50_MOD_I32 8 + +/* NV50_MOD_I32 is used to indicate integer mode for neg/abs */ + +/* STACK: Conditionals and loops have to use the (per warp) stack. + * Stack entries consist of an entry type (divergent path, join at), + * a mask indicating the active threads of the warp, and an address. + * MPs can store 12 stack entries internally, if we need more (and + * we probably do), we have to create a stack buffer in VRAM. + */ +/* impose low limits for now */ +#define NV50_MAX_COND_NESTING 4 +#define NV50_MAX_LOOP_NESTING 3 + +#define JOIN_ON(e) e; pc->p->exec_tail->inst[1] |= 2 struct nv50_pc { struct nv50_program *p; /* hw resources */ struct nv50_reg *r_temp[NV50_SU_MAX_TEMP]; + struct nv50_reg r_addr[NV50_SU_MAX_ADDR]; /* tgsi resources */ struct nv50_reg *temp; @@ -110,8 +131,11 @@ struct nv50_pc { struct nv50_reg *param; int param_nr; struct nv50_reg *immd; - float *immd_buf; + uint32_t *immd_buf; int immd_nr; + struct nv50_reg **addr; + int addr_nr; + uint8_t addr_alloc; /* set bit indicates used for TGSI_FILE_ADDRESS */ struct nv50_reg *temp_temp[16]; unsigned temp_temp_nr; @@ -120,23 +144,30 @@ struct nv50_pc { struct nv50_reg *r_brdc; struct nv50_reg *r_dst[4]; + struct nv50_reg reg_instances[16]; + unsigned reg_instance_nr; + unsigned interp_mode[32]; /* perspective interpolation registers */ struct nv50_reg *iv_p; struct nv50_reg *iv_c; - struct nv50_program_exec *if_cond; - struct nv50_program_exec *if_insn[MAX_IF_DEPTH]; - struct nv50_program_exec *br_join[MAX_IF_DEPTH]; - struct nv50_program_exec *br_loop[MAX_LOOP_DEPTH]; /* for BRK branch */ + struct nv50_program_exec *if_insn[NV50_MAX_COND_NESTING]; + struct nv50_program_exec *if_join[NV50_MAX_COND_NESTING]; + struct nv50_program_exec *loop_brka[NV50_MAX_LOOP_NESTING]; int if_lvl, loop_lvl; - unsigned loop_pos[MAX_LOOP_DEPTH]; + unsigned loop_pos[NV50_MAX_LOOP_NESTING]; + + unsigned *insn_pos; /* actual program offset of each TGSI insn */ + boolean in_subroutine; /* current instruction and total number of insns */ unsigned insn_cur; unsigned insn_nr; boolean allow32; + + uint8_t edgeflag_out; }; static INLINE void @@ -145,7 +176,7 @@ ctor_reg(struct nv50_reg *reg, unsigned type, int index, int hw) reg->type = type; reg->index = index; reg->hw = hw; - reg->neg = 0; + reg->mod = 0; reg->rhw = -1; reg->acc = 0; } @@ -159,6 +190,16 @@ popcnt4(uint32_t val) } static void +terminate_mbb(struct nv50_pc *pc) +{ + int i; + + /* remove records of temporary address register values */ + for (i = 0; i < NV50_SU_MAX_ADDR; ++i) + pc->r_addr[i].rhw = -1; +} + +static void alloc_reg(struct nv50_pc *pc, struct nv50_reg *reg) { int i = 0; @@ -207,6 +248,21 @@ alloc_reg(struct nv50_pc *pc, struct nv50_reg *reg) assert(0); } +static INLINE struct nv50_reg * +reg_instance(struct nv50_pc *pc, struct nv50_reg *reg) +{ + struct nv50_reg *ri; + + assert(pc->reg_instance_nr < 16); + ri = &pc->reg_instances[pc->reg_instance_nr++]; + if (reg) { + alloc_reg(pc, reg); + *ri = *reg; + reg->mod = 0; + } + return ri; +} + /* XXX: For shaders that aren't executed linearly (e.g. shaders that * contain loops), we need to assign all hw regs to TGSI TEMPs early, * lest we risk temp_temps overwriting regs alloc'd "later". @@ -233,22 +289,6 @@ alloc_temp(struct nv50_pc *pc, struct nv50_reg *dst) return NULL; } -/* Assign the hw of the discarded temporary register src - * to the tgsi register dst and free src. - */ -static void -assimilate_temp(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) -{ - assert(src->index == -1 && src->hw != -1); - - if (dst->hw != -1) - pc->r_temp[dst->hw] = NULL; - pc->r_temp[src->hw] = dst; - dst->hw = src->hw; - - FREE(src); -} - /* release the hardware resource held by r */ static void release_hw(struct nv50_pc *pc, struct nv50_reg *r) @@ -320,25 +360,34 @@ static void kill_temp_temp(struct nv50_pc *pc) { int i; - + for (i = 0; i < pc->temp_temp_nr; i++) free_temp(pc, pc->temp_temp[i]); pc->temp_temp_nr = 0; } static int -ctor_immd(struct nv50_pc *pc, float x, float y, float z, float w) +ctor_immd_4u32(struct nv50_pc *pc, + uint32_t x, uint32_t y, uint32_t z, uint32_t w) { - pc->immd_buf = REALLOC(pc->immd_buf, (pc->immd_nr * 4 * sizeof(float)), - (pc->immd_nr + 1) * 4 * sizeof(float)); + unsigned size = pc->immd_nr * 4 * sizeof(uint32_t); + + pc->immd_buf = REALLOC(pc->immd_buf, size, size + 4 * sizeof(uint32_t)); + pc->immd_buf[(pc->immd_nr * 4) + 0] = x; pc->immd_buf[(pc->immd_nr * 4) + 1] = y; pc->immd_buf[(pc->immd_nr * 4) + 2] = z; pc->immd_buf[(pc->immd_nr * 4) + 3] = w; - + return pc->immd_nr++; } +static INLINE int +ctor_immd_4f32(struct nv50_pc *pc, float x, float y, float z, float w) +{ + return ctor_immd_4u32(pc, fui(x), fui(y), fui(z), fui(w)); +} + static struct nv50_reg * alloc_immd(struct nv50_pc *pc, float f) { @@ -346,11 +395,11 @@ alloc_immd(struct nv50_pc *pc, float f) unsigned hw; for (hw = 0; hw < pc->immd_nr * 4; hw++) - if (pc->immd_buf[hw] == f) + if (pc->immd_buf[hw] == fui(f)) break; if (hw == pc->immd_nr * 4) - hw = ctor_immd(pc, f, -f, 0.5 * f, 0) * 4; + hw = ctor_immd_4f32(pc, f, -f, 0.5 * f, 0) * 4; ctor_reg(r, P_IMMD, -1, hw); return r; @@ -396,10 +445,19 @@ is_immd(struct nv50_program_exec *e) return FALSE; } +static boolean +is_join(struct nv50_program_exec *e) +{ + if (is_long(e) && (e->inst[1] & 3) == 2) + return TRUE; + return FALSE; +} + static INLINE void set_pred(struct nv50_pc *pc, unsigned pred, unsigned idx, struct nv50_program_exec *e) { + assert(!is_immd(e)); set_long(pc, e); e->inst[1] &= ~((0x1f << 7) | (0x3 << 12)); e->inst[1] |= (pred << 7) | (idx << 12); @@ -434,29 +492,119 @@ set_dst(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_program_exec *e) } alloc_reg(pc, dst); + if (dst->hw > 63) + set_long(pc, e); e->inst[0] |= (dst->hw << 2); } static INLINE void set_immd(struct nv50_pc *pc, struct nv50_reg *imm, struct nv50_program_exec *e) { - float f = pc->immd_buf[imm->hw]; - unsigned val = fui(imm->neg ? -f : f); - set_long(pc, e); - /*XXX: can't be predicated - bits overlap.. catch cases where both - * are required and avoid them. */ + /* XXX: can't be predicated - bits overlap; cases where both + * are required should be avoided by using pc->allow32 */ set_pred(pc, 0, 0, e); set_pred_wr(pc, 0, 0, e); e->inst[1] |= 0x00000002 | 0x00000001; - e->inst[0] |= (val & 0x3f) << 16; - e->inst[1] |= (val >> 6) << 2; + e->inst[0] |= (pc->immd_buf[imm->hw] & 0x3f) << 16; + e->inst[1] |= (pc->immd_buf[imm->hw] >> 6) << 2; } +static INLINE void +set_addr(struct nv50_program_exec *e, struct nv50_reg *a) +{ + assert(!(e->inst[0] & 0x0c000000)); + assert(!(e->inst[1] & 0x00000004)); + + e->inst[0] |= (a->hw & 3) << 26; + e->inst[1] |= (a->hw >> 2) << 2; +} + +static void +emit_add_addr_imm(struct nv50_pc *pc, struct nv50_reg *dst, + struct nv50_reg *src0, uint16_t src1_val) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0xd0000000 | (src1_val << 9); + e->inst[1] = 0x20000000; + set_long(pc, e); + e->inst[0] |= dst->hw << 2; + if (src0) /* otherwise will add to $a0, which is always 0 */ + set_addr(e, src0); + + emit(pc, e); +} + +static struct nv50_reg * +alloc_addr(struct nv50_pc *pc, struct nv50_reg *ref) +{ + struct nv50_reg *a_tgsi = NULL, *a = NULL; + int i; + uint8_t avail = ~pc->addr_alloc; + + if (!ref) { + /* allocate for TGSI_FILE_ADDRESS */ + while (avail) { + i = ffs(avail) - 1; + + if (pc->r_addr[i].rhw < 0 || + pc->r_addr[i].acc != pc->insn_cur) { + pc->addr_alloc |= (1 << i); + + pc->r_addr[i].rhw = -1; + pc->r_addr[i].index = i; + return &pc->r_addr[i]; + } + avail &= ~(1 << i); + } + assert(0); + return NULL; + } + + /* Allocate and set an address reg so we can access 'ref'. + * + * If and r_addr->index will be -1 or the hw index the value + * value in rhw is relative to. If rhw < 0, the reg has not + * been initialized or is in use for TGSI_FILE_ADDRESS. + */ + while (avail) { /* only consider regs that are not TGSI */ + i = ffs(avail) - 1; + avail &= ~(1 << i); + + if ((!a || a->rhw >= 0) && pc->r_addr[i].rhw < 0) { + /* prefer an usused reg with low hw index */ + a = &pc->r_addr[i]; + continue; + } + if (!a && pc->r_addr[i].acc != pc->insn_cur) + a = &pc->r_addr[i]; + + if (ref->hw - pc->r_addr[i].rhw >= 128) + continue; + + if ((ref->acc >= 0 && pc->r_addr[i].index < 0) || + (ref->acc < 0 && pc->r_addr[i].index == ref->index)) { + pc->r_addr[i].acc = pc->insn_cur; + return &pc->r_addr[i]; + } + } + assert(a); + + if (ref->acc < 0) + a_tgsi = pc->addr[ref->index]; + + emit_add_addr_imm(pc, a, a_tgsi, (ref->hw & ~0x7f) * 4); + + a->rhw = ref->hw & ~0x7f; + a->acc = pc->insn_cur; + a->index = a_tgsi ? ref->index : -1; + return a; +} #define INTERP_LINEAR 0 -#define INTERP_FLAT 1 +#define INTERP_FLAT 1 #define INTERP_PERSPECTIVE 2 #define INTERP_CENTROID 4 @@ -494,23 +642,34 @@ set_data(struct nv50_pc *pc, struct nv50_reg *src, unsigned m, unsigned s, { set_long(pc, e); - e->param.index = src->hw; + e->param.index = src->hw & 127; e->param.shift = s; e->param.mask = m << (s % 32); + if (src->hw > 127) + set_addr(e, alloc_addr(pc, src)); + else + if (src->acc < 0) { + assert(src->type == P_CONST); + set_addr(e, pc->addr[src->index]); + } + e->inst[1] |= (((src->type == P_IMMD) ? 0 : 1) << 22); } +/* Never apply nv50_reg::mod in emit_mov, or carefully check the code !!! */ static void emit_mov(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) { struct nv50_program_exec *e = exec(pc); - e->inst[0] |= 0x10000000; + e->inst[0] = 0x10000000; + if (!pc->allow32) + set_long(pc, e); set_dst(pc, dst, e); - if (pc->allow32 && dst->type != P_RESULT && src->type == P_IMMD) { + if (!is_long(e) && src->type == P_IMMD) { set_immd(pc, src, e); /*XXX: 32-bit, but steals part of "half" reg space - need to * catch and handle this case if/when we do half-regs @@ -519,7 +678,7 @@ emit_mov(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) if (src->type == P_IMMD || src->type == P_CONST) { set_long(pc, e); set_data(pc, src, 0x7f, 9, e); - e->inst[1] |= 0x20000000; /* src0 const? */ + e->inst[1] |= 0x20000000; /* mov from c[] */ } else { if (src->type == P_ATTR) { set_long(pc, e); @@ -527,14 +686,16 @@ emit_mov(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) } alloc_reg(pc, src); + if (src->hw > 63) + set_long(pc, e); e->inst[0] |= (src->hw << 9); } if (is_long(e) && !is_immd(e)) { e->inst[1] |= 0x04000000; /* 32-bit */ - e->inst[1] |= 0x0000c000; /* "subsubop" 0x3 */ + e->inst[1] |= 0x0000c000; /* 32-bit c[] load / lane mask 0:1 */ if (!(e->inst[1] & 0x20000000)) - e->inst[1] |= 0x00030000; /* "subsubop" 0xf */ + e->inst[1] |= 0x00030000; /* lane mask 2:3 */ } else e->inst[0] |= 0x00008000; @@ -549,6 +710,45 @@ emit_mov_immdval(struct nv50_pc *pc, struct nv50_reg *dst, float f) FREE(imm); } +/* Assign the hw of the discarded temporary register src + * to the tgsi register dst and free src. + */ +static void +assimilate_temp(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) +{ + assert(src->index == -1 && src->hw != -1); + + if (pc->if_lvl || pc->loop_lvl || + (dst->type != P_TEMP) || + (src->hw < pc->result_nr * 4 && + pc->p->type == PIPE_SHADER_FRAGMENT) || + pc->p->info.opcode_count[TGSI_OPCODE_CAL] || + pc->p->info.opcode_count[TGSI_OPCODE_BRA]) { + + emit_mov(pc, dst, src); + free_temp(pc, src); + return; + } + + if (dst->hw != -1) + pc->r_temp[dst->hw] = NULL; + pc->r_temp[src->hw] = dst; + dst->hw = src->hw; + + FREE(src); +} + +static void +emit_nop(struct nv50_pc *pc) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0xf0000000; + set_long(pc, e); + e->inst[1] = 0xe0000000; + emit(pc, e); +} + static boolean check_swap_src_0_1(struct nv50_pc *pc, struct nv50_reg **s0, struct nv50_reg **s1) @@ -586,6 +786,8 @@ set_src_0_restricted(struct nv50_pc *pc, struct nv50_reg *src, } alloc_reg(pc, src); + if (src->hw > 63) + set_long(pc, e); e->inst[0] |= (src->hw << 9); } @@ -604,6 +806,8 @@ set_src_0(struct nv50_pc *pc, struct nv50_reg *src, struct nv50_program_exec *e) } alloc_reg(pc, src); + if (src->hw > 63) + set_long(pc, e); e->inst[0] |= (src->hw << 9); } @@ -630,7 +834,9 @@ set_src_1(struct nv50_pc *pc, struct nv50_reg *src, struct nv50_program_exec *e) } alloc_reg(pc, src); - e->inst[0] |= (src->hw << 16); + if (src->hw > 63) + set_long(pc, e); + e->inst[0] |= ((src->hw & 127) << 16); } static void @@ -658,7 +864,34 @@ set_src_2(struct nv50_pc *pc, struct nv50_reg *src, struct nv50_program_exec *e) } alloc_reg(pc, src); - e->inst[1] |= (src->hw << 14); + e->inst[1] |= ((src->hw & 127) << 14); +} + +static void +emit_mov_from_pred(struct nv50_pc *pc, struct nv50_reg *dst, int pred) +{ + struct nv50_program_exec *e = exec(pc); + + assert(dst->type == P_TEMP); + e->inst[1] = 0x20000000 | (pred << 12); + set_long(pc, e); + set_dst(pc, dst, e); + + emit(pc, e); +} + +static void +emit_mov_to_pred(struct nv50_pc *pc, int pred, struct nv50_reg *src) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0x000001fc; + e->inst[1] = 0xa0000008; + set_long(pc, e); + set_pred_wr(pc, 1, pred, e); + set_src_0_restricted(pc, src, e); + + emit(pc, e); } static void @@ -676,12 +909,12 @@ emit_mul(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src0, set_dst(pc, dst, e); set_src_0(pc, src0, e); if (src1->type == P_IMMD && !is_long(e)) { - if (src0->neg) + if (src0->mod ^ src1->mod) e->inst[0] |= 0x00008000; set_immd(pc, src1, e); } else { set_src_1(pc, src1, e); - if (src0->neg ^ src1->neg) { + if ((src0->mod ^ src1->mod) & NV50_MOD_NEG) { if (is_long(e)) e->inst[1] |= 0x08000000; else @@ -698,13 +931,15 @@ emit_add(struct nv50_pc *pc, struct nv50_reg *dst, { struct nv50_program_exec *e = exec(pc); - e->inst[0] |= 0xb0000000; + e->inst[0] = 0xb0000000; + alloc_reg(pc, src1); check_swap_src_0_1(pc, &src0, &src1); - if (!pc->allow32 || src0->neg || src1->neg) { + if (!pc->allow32 || (src0->mod | src1->mod) || src1->hw > 63) { set_long(pc, e); - e->inst[1] |= (src0->neg << 26) | (src1->neg << 27); + e->inst[1] |= ((src0->mod & NV50_MOD_NEG) << 26) | + ((src1->mod & NV50_MOD_NEG) << 27); } set_dst(pc, dst, e); @@ -721,20 +956,48 @@ emit_add(struct nv50_pc *pc, struct nv50_reg *dst, } static void +emit_arl(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src, + uint8_t s) +{ + struct nv50_program_exec *e = exec(pc); + + set_long(pc, e); + e->inst[1] |= 0xc0000000; + + e->inst[0] |= dst->hw << 2; + e->inst[0] |= s << 16; /* shift left */ + set_src_0_restricted(pc, src, e); + + emit(pc, e); +} + +#define NV50_MAX_F32 0x880 +#define NV50_MAX_S32 0x08c +#define NV50_MAX_U32 0x084 +#define NV50_MIN_F32 0x8a0 +#define NV50_MIN_S32 0x0ac +#define NV50_MIN_U32 0x0a4 + +static void emit_minmax(struct nv50_pc *pc, unsigned sub, struct nv50_reg *dst, struct nv50_reg *src0, struct nv50_reg *src1) { struct nv50_program_exec *e = exec(pc); set_long(pc, e); - e->inst[0] |= 0xb0000000; - e->inst[1] |= (sub << 29); + e->inst[0] |= 0x30000000 | ((sub & 0x800) << 20); + e->inst[1] |= (sub << 24); check_swap_src_0_1(pc, &src0, &src1); set_dst(pc, dst, e); set_src_0(pc, src0, e); set_src_1(pc, src1, e); + if (src0->mod & NV50_MOD_ABS) + e->inst[1] |= 0x00100000; + if (src1->mod & NV50_MOD_ABS) + e->inst[1] |= 0x00080000; + emit(pc, e); } @@ -742,9 +1005,76 @@ static INLINE void emit_sub(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src0, struct nv50_reg *src1) { - src1->neg ^= 1; + src1->mod ^= NV50_MOD_NEG; emit_add(pc, dst, src0, src1); - src1->neg ^= 1; + src1->mod ^= NV50_MOD_NEG; +} + +static void +emit_bitop2(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src0, + struct nv50_reg *src1, unsigned op) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0xd0000000; + set_long(pc, e); + + check_swap_src_0_1(pc, &src0, &src1); + set_dst(pc, dst, e); + set_src_0(pc, src0, e); + + if (op != TGSI_OPCODE_AND && op != TGSI_OPCODE_OR && + op != TGSI_OPCODE_XOR) + assert(!"invalid bit op"); + + assert(!(src0->mod | src1->mod)); + + if (src1->type == P_IMMD && src0->type == P_TEMP && pc->allow32) { + set_immd(pc, src1, e); + if (op == TGSI_OPCODE_OR) + e->inst[0] |= 0x0100; + else + if (op == TGSI_OPCODE_XOR) + e->inst[0] |= 0x8000; + } else { + set_src_1(pc, src1, e); + e->inst[1] |= 0x04000000; /* 32 bit */ + if (op == TGSI_OPCODE_OR) + e->inst[1] |= 0x4000; + else + if (op == TGSI_OPCODE_XOR) + e->inst[1] |= 0x8000; + } + + emit(pc, e); +} + +static void +emit_shift(struct nv50_pc *pc, struct nv50_reg *dst, + struct nv50_reg *src0, struct nv50_reg *src1, unsigned dir) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0x30000000; + e->inst[1] = 0xc4000000; + + set_long(pc, e); + set_dst(pc, dst, e); + set_src_0(pc, src0, e); + + if (src1->type == P_IMMD) { + e->inst[1] |= (1 << 20); + e->inst[0] |= (pc->immd_buf[src1->hw] & 0x7f) << 16; + } else + set_src_1(pc, src1, e); + + if (dir != TGSI_OPCODE_SHL) + e->inst[1] |= (1 << 29); + + if (dir == TGSI_OPCODE_ISHR) + e->inst[1] |= (1 << 27); + + emit(pc, e); } static void @@ -761,9 +1091,9 @@ emit_mad(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src0, set_src_1(pc, src1, e); set_src_2(pc, src2, e); - if (src0->neg ^ src1->neg) + if ((src0->mod ^ src1->mod) & NV50_MOD_NEG) e->inst[1] |= 0x04000000; - if (src2->neg) + if (src2->mod & NV50_MOD_NEG) e->inst[1] |= 0x08000000; emit(pc, e); @@ -773,11 +1103,19 @@ static INLINE void emit_msb(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src0, struct nv50_reg *src1, struct nv50_reg *src2) { - src2->neg ^= 1; + src2->mod ^= NV50_MOD_NEG; emit_mad(pc, dst, src0, src1, src2); - src2->neg ^= 1; + src2->mod ^= NV50_MOD_NEG; } +#define NV50_FLOP_RCP 0 +#define NV50_FLOP_RSQ 2 +#define NV50_FLOP_LG2 3 +#define NV50_FLOP_SIN 4 +#define NV50_FLOP_COS 5 +#define NV50_FLOP_EX2 6 + +/* rcp, rsqrt, lg2 support neg and abs */ static void emit_flop(struct nv50_pc *pc, unsigned sub, struct nv50_reg *dst, struct nv50_reg *src) @@ -785,17 +1123,20 @@ emit_flop(struct nv50_pc *pc, unsigned sub, struct nv50_program_exec *e = exec(pc); e->inst[0] |= 0x90000000; - if (sub) { + if (sub || src->mod) { set_long(pc, e); e->inst[1] |= (sub << 29); } set_dst(pc, dst, e); + set_src_0_restricted(pc, src, e); - if (sub == 0 || sub == 2) - set_src_0_restricted(pc, src, e); - else - set_src_0(pc, src, e); + assert(!src->mod || sub < 4); + + if (src->mod & NV50_MOD_NEG) + e->inst[1] |= 0x04000000; + if (src->mod & NV50_MOD_ABS) + e->inst[1] |= 0x00100000; emit(pc, e); } @@ -812,6 +1153,11 @@ emit_preex2(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) set_long(pc, e); e->inst[1] |= (6 << 29) | 0x00004000; + if (src->mod & NV50_MOD_NEG) + e->inst[1] |= 0x04000000; + if (src->mod & NV50_MOD_ABS) + e->inst[1] |= 0x00100000; + emit(pc, e); } @@ -827,40 +1173,49 @@ emit_precossin(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) set_long(pc, e); e->inst[1] |= (6 << 29); + if (src->mod & NV50_MOD_NEG) + e->inst[1] |= 0x04000000; + if (src->mod & NV50_MOD_ABS) + e->inst[1] |= 0x00100000; + emit(pc, e); } -#define CVTOP_RN 0x01 -#define CVTOP_FLOOR 0x03 -#define CVTOP_CEIL 0x05 -#define CVTOP_TRUNC 0x07 -#define CVTOP_SAT 0x08 -#define CVTOP_ABS 0x10 - -/* 0x04 == 32 bit */ -/* 0x40 == dst is float */ -/* 0x80 == src is float */ -#define CVT_F32_F32 0xc4 -#define CVT_F32_S32 0x44 -#define CVT_F32_U32 0x64 -#define CVT_S32_F32 0x8c -#define CVT_S32_S32 0x0c -#define CVT_NEG 0x20 -#define CVT_RI 0x08 +#define CVT_RN (0x00 << 16) +#define CVT_FLOOR (0x02 << 16) +#define CVT_CEIL (0x04 << 16) +#define CVT_TRUNC (0x06 << 16) +#define CVT_SAT (0x08 << 16) +#define CVT_ABS (0x10 << 16) + +#define CVT_X32_X32 0x04004000 +#define CVT_X32_S32 0x04014000 +#define CVT_F32_F32 ((0xc0 << 24) | CVT_X32_X32) +#define CVT_S32_F32 ((0x88 << 24) | CVT_X32_X32) +#define CVT_U32_F32 ((0x80 << 24) | CVT_X32_X32) +#define CVT_F32_S32 ((0x40 << 24) | CVT_X32_S32) +#define CVT_F32_U32 ((0x40 << 24) | CVT_X32_X32) +#define CVT_S32_S32 ((0x08 << 24) | CVT_X32_S32) +#define CVT_S32_U32 ((0x08 << 24) | CVT_X32_X32) +#define CVT_U32_S32 ((0x00 << 24) | CVT_X32_S32) + +#define CVT_NEG 0x20000000 +#define CVT_RI 0x08000000 static void emit_cvt(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src, - int wp, unsigned cvn, unsigned fmt) + int wp, uint32_t cvn) { struct nv50_program_exec *e; e = exec(pc); - set_long(pc, e); - e->inst[0] |= 0xa0000000; - e->inst[1] |= 0x00004000; - e->inst[1] |= (cvn << 16); - e->inst[1] |= (fmt << 24); + if (src->mod & NV50_MOD_NEG) cvn |= CVT_NEG; + if (src->mod & NV50_MOD_ABS) cvn |= CVT_ABS; + + e->inst[0] = 0xa0000000; + e->inst[1] = cvn; + set_long(pc, e); set_src_0(pc, src, e); if (wp >= 0) @@ -885,10 +1240,12 @@ emit_cvt(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src, * 0x6 = GE * 0x7 = set condition code ? (used before bra.lt/le/gt/ge) * 0x8 = unordered bit (allows NaN) + * + * mode = 0x04 (u32), 0x0c (s32), 0x80 (f32) */ static void emit_set(struct nv50_pc *pc, unsigned ccode, struct nv50_reg *dst, int wp, - struct nv50_reg *src0, struct nv50_reg *src1) + struct nv50_reg *src0, struct nv50_reg *src1, uint8_t mode) { static const unsigned cc_swapped[8] = { 0, 4, 2, 6, 1, 5, 3, 7 }; @@ -903,16 +1260,10 @@ emit_set(struct nv50_pc *pc, unsigned ccode, struct nv50_reg *dst, int wp, if (dst && dst->type != P_TEMP) dst = alloc_temp(pc, NULL); - /* set.u32 */ set_long(pc, e); - e->inst[0] |= 0xb0000000; + e->inst[0] |= 0x30000000 | (mode << 24); e->inst[1] |= 0x60000000 | (ccode << 14); - /* XXX: decuda will disasm as .u16 and use .lo/.hi regs, but - * that doesn't seem to match what the hw actually does - e->inst[1] |= 0x04000000; << breaks things, u32 by default ? - */ - if (wp >= 0) set_pred_wr(pc, 1, wp, e); if (dst) @@ -926,35 +1277,108 @@ emit_set(struct nv50_pc *pc, unsigned ccode, struct nv50_reg *dst, int wp, set_src_1(pc, src1, e); emit(pc, e); - pc->if_cond = pc->p->exec_tail; /* record for OPCODE_IF */ - /* cvt.f32.u32/s32 (?) if we didn't only write the predicate */ - if (rdst) - emit_cvt(pc, rdst, dst, -1, CVTOP_ABS | CVTOP_RN, CVT_F32_S32); + if (rdst && mode == 0x80) /* convert to float ? */ + emit_cvt(pc, rdst, dst, -1, CVT_ABS | CVT_F32_S32); if (rdst && rdst != dst) free_temp(pc, dst); } -static INLINE unsigned -map_tgsi_setop_cc(unsigned op) +static INLINE void +map_tgsi_setop_hw(unsigned op, uint8_t *cc, uint8_t *ty) { switch (op) { - case TGSI_OPCODE_SLT: return 0x1; - case TGSI_OPCODE_SGE: return 0x6; - case TGSI_OPCODE_SEQ: return 0x2; - case TGSI_OPCODE_SGT: return 0x4; - case TGSI_OPCODE_SLE: return 0x3; - case TGSI_OPCODE_SNE: return 0xd; + case TGSI_OPCODE_SLT: *cc = 0x1; *ty = 0x80; break; + case TGSI_OPCODE_SGE: *cc = 0x6; *ty = 0x80; break; + case TGSI_OPCODE_SEQ: *cc = 0x2; *ty = 0x80; break; + case TGSI_OPCODE_SGT: *cc = 0x4; *ty = 0x80; break; + case TGSI_OPCODE_SLE: *cc = 0x3; *ty = 0x80; break; + case TGSI_OPCODE_SNE: *cc = 0xd; *ty = 0x80; break; + + case TGSI_OPCODE_ISLT: *cc = 0x1; *ty = 0x0c; break; + case TGSI_OPCODE_ISGE: *cc = 0x6; *ty = 0x0c; break; + case TGSI_OPCODE_USEQ: *cc = 0x2; *ty = 0x04; break; + case TGSI_OPCODE_USGE: *cc = 0x6; *ty = 0x04; break; + case TGSI_OPCODE_USLT: *cc = 0x1; *ty = 0x04; break; + case TGSI_OPCODE_USNE: *cc = 0x5; *ty = 0x04; break; default: assert(0); - return 0; + return; } } +static void +emit_add_b32(struct nv50_pc *pc, struct nv50_reg *dst, + struct nv50_reg *src0, struct nv50_reg *rsrc1) +{ + struct nv50_program_exec *e = exec(pc); + struct nv50_reg *src1; + + e->inst[0] = 0x20000000; + + alloc_reg(pc, rsrc1); + check_swap_src_0_1(pc, &src0, &rsrc1); + + src1 = rsrc1; + if (src0->mod & rsrc1->mod & NV50_MOD_NEG) { + src1 = alloc_temp(pc, NULL); + emit_cvt(pc, src1, rsrc1, -1, CVT_S32_S32); + } + + if (!pc->allow32 || src1->hw > 63 || + (src1->type != P_TEMP && src1->type != P_IMMD)) + set_long(pc, e); + + set_dst(pc, dst, e); + set_src_0(pc, src0, e); + + if (is_long(e)) { + e->inst[1] |= 1 << 26; + set_src_2(pc, src1, e); + } else { + e->inst[0] |= 0x8000; + if (src1->type == P_IMMD) + set_immd(pc, src1, e); + else + set_src_1(pc, src1, e); + } + + if (src0->mod & NV50_MOD_NEG) + e->inst[0] |= 1 << 28; + else + if (src1->mod & NV50_MOD_NEG) + e->inst[0] |= 1 << 22; + + emit(pc, e); + + if (src1 != rsrc1) + free_temp(pc, src1); +} + +static void +emit_sad(struct nv50_pc *pc, struct nv50_reg *dst, + struct nv50_reg *src0, struct nv50_reg *src1, struct nv50_reg *src2) +{ + struct nv50_program_exec *e = exec(pc); + + e->inst[0] = 0x50000000; + set_dst(pc, dst, e); + set_src_0(pc, src0, e); + set_src_1(pc, src1, e); + alloc_reg(pc, src2); + if (is_long(e) || (src2->type != dst->type) || (src2->hw != dst->hw)) + set_src_2(pc, src2, e); + + if (is_long(e)) + e->inst[1] |= 0x0c << 24; + else + e->inst[0] |= 0x81 << 8; +} + static INLINE void emit_flr(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) { - emit_cvt(pc, dst, src, -1, CVTOP_FLOOR, CVT_F32_F32 | CVT_RI); + emit_cvt(pc, dst, src, -1, CVT_FLOOR | CVT_F32_F32 | CVT_RI); } static void @@ -963,24 +1387,18 @@ emit_pow(struct nv50_pc *pc, struct nv50_reg *dst, { struct nv50_reg *temp = alloc_temp(pc, NULL); - emit_flop(pc, 3, temp, v); + emit_flop(pc, NV50_FLOP_LG2, temp, v); emit_mul(pc, temp, temp, e); emit_preex2(pc, temp, temp); - emit_flop(pc, 6, dst, temp); + emit_flop(pc, NV50_FLOP_EX2, dst, temp); free_temp(pc, temp); } static INLINE void -emit_abs(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) -{ - emit_cvt(pc, dst, src, -1, CVTOP_ABS, CVT_F32_F32); -} - -static INLINE void emit_sat(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) { - emit_cvt(pc, dst, src, -1, CVTOP_SAT, CVT_F32_F32); + emit_cvt(pc, dst, src, -1, CVT_SAT | CVT_F32_F32); } static void @@ -998,18 +1416,18 @@ emit_lit(struct nv50_pc *pc, struct nv50_reg **dst, unsigned mask, if (mask & (3 << 1)) { tmp[0] = alloc_temp(pc, NULL); - emit_minmax(pc, 4, tmp[0], src[0], zero); + emit_minmax(pc, NV50_MAX_F32, tmp[0], src[0], zero); } if (mask & (1 << 2)) { set_pred_wr(pc, 1, 0, pc->p->exec_tail); tmp[1] = temp_temp(pc); - emit_minmax(pc, 4, tmp[1], src[1], zero); + emit_minmax(pc, NV50_MAX_F32, tmp[1], src[1], zero); tmp[3] = temp_temp(pc); - emit_minmax(pc, 4, tmp[3], src[3], neg128); - emit_minmax(pc, 5, tmp[3], tmp[3], pos128); + emit_minmax(pc, NV50_MAX_F32, tmp[3], src[3], neg128); + emit_minmax(pc, NV50_MIN_F32, tmp[3], tmp[3], pos128); emit_pow(pc, dst[2], tmp[1], tmp[3]); emit_mov(pc, dst[2], zero); @@ -1038,132 +1456,356 @@ emit_lit(struct nv50_pc *pc, struct nv50_reg **dst, unsigned mask, } static void -emit_neg(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) +emit_kil(struct nv50_pc *pc, struct nv50_reg *src) +{ + struct nv50_program_exec *e; + const int r_pred = 1; + + e = exec(pc); + e->inst[0] = 0x00000002; /* discard */ + set_long(pc, e); /* sets cond code to ALWAYS */ + + if (src) { + set_pred(pc, 0x1 /* cc = LT */, r_pred, e); + /* write to predicate reg */ + emit_cvt(pc, NULL, src, r_pred, CVT_F32_F32); + } + + emit(pc, e); +} + +static struct nv50_program_exec * +emit_control_flow(struct nv50_pc *pc, unsigned op, int pred, unsigned cc) { struct nv50_program_exec *e = exec(pc); + e->inst[0] = (op << 28) | 2; set_long(pc, e); - e->inst[0] |= 0xa0000000; /* delta */ - e->inst[1] |= (7 << 29); /* delta */ - e->inst[1] |= 0x04000000; /* negate arg0? probably not */ - e->inst[1] |= (1 << 14); /* src .f32 */ - set_dst(pc, dst, e); - set_src_0(pc, src, e); + if (pred >= 0) + set_pred(pc, cc, pred, e); emit(pc, e); + return e; +} + +static INLINE struct nv50_program_exec * +emit_breakaddr(struct nv50_pc *pc) +{ + return emit_control_flow(pc, 0x4, -1, 0); +} + +static INLINE void +emit_break(struct nv50_pc *pc, int pred, unsigned cc) +{ + emit_control_flow(pc, 0x5, pred, cc); +} + +static INLINE struct nv50_program_exec * +emit_joinat(struct nv50_pc *pc) +{ + return emit_control_flow(pc, 0xa, -1, 0); } +static INLINE struct nv50_program_exec * +emit_branch(struct nv50_pc *pc, int pred, unsigned cc) +{ + return emit_control_flow(pc, 0x1, pred, cc); +} + +static INLINE struct nv50_program_exec * +emit_call(struct nv50_pc *pc, int pred, unsigned cc) +{ + return emit_control_flow(pc, 0x2, pred, cc); +} + +static INLINE void +emit_ret(struct nv50_pc *pc, int pred, unsigned cc) +{ + emit_control_flow(pc, 0x3, pred, cc); +} + +#define QOP_ADD 0 +#define QOP_SUBR 1 +#define QOP_SUB 2 +#define QOP_MOV_SRC1 3 + +/* For a quad of threads / top left, top right, bottom left, bottom right + * pixels, do a different operation, and take src0 from a specific thread. + */ static void -emit_kil(struct nv50_pc *pc, struct nv50_reg *src) +emit_quadop(struct nv50_pc *pc, struct nv50_reg *dst, int wp, int lane_src0, + struct nv50_reg *src0, struct nv50_reg *src1, ubyte qop) { - struct nv50_program_exec *e; - const int r_pred = 1; + struct nv50_program_exec *e = exec(pc); - /* Sets predicate reg ? */ - e = exec(pc); - e->inst[0] = 0xa00001fd; - e->inst[1] = 0xc4014788; - set_src_0(pc, src, e); - set_pred_wr(pc, 1, r_pred, e); - if (src->neg) - e->inst[1] |= 0x20000000; - emit(pc, e); + e->inst[0] = 0xc0000000; + e->inst[1] = 0x80000000; + set_long(pc, e); + e->inst[0] |= lane_src0 << 16; + set_src_0(pc, src0, e); + set_src_2(pc, src1, e); - /* This is probably KILP */ - e = exec(pc); - e->inst[0] = 0x000001fe; - set_long(pc, e); - set_pred(pc, 1 /* LT? */, r_pred, e); - emit(pc, e); + if (wp >= 0) + set_pred_wr(pc, 1, wp, e); + + if (dst) + set_dst(pc, dst, e); + else { + e->inst[0] |= 0x000001fc; + e->inst[1] |= 0x00000008; + } + + e->inst[0] |= (qop & 3) << 20; + e->inst[1] |= (qop >> 2) << 22; + + emit(pc, e); } static void -emit_tex(struct nv50_pc *pc, struct nv50_reg **dst, unsigned mask, - struct nv50_reg **src, unsigned unit, unsigned type, boolean proj) +load_cube_tex_coords(struct nv50_pc *pc, struct nv50_reg *t[4], + struct nv50_reg **src, unsigned arg, boolean proj) { - struct nv50_reg *temp, *t[4]; - struct nv50_program_exec *e; + int mod[3] = { src[0]->mod, src[1]->mod, src[2]->mod }; + + src[0]->mod |= NV50_MOD_ABS; + src[1]->mod |= NV50_MOD_ABS; + src[2]->mod |= NV50_MOD_ABS; + + emit_minmax(pc, NV50_MAX_F32, t[2], src[0], src[1]); + emit_minmax(pc, NV50_MAX_F32, t[2], src[2], t[2]); + + src[0]->mod = mod[0]; + src[1]->mod = mod[1]; + src[2]->mod = mod[2]; + + if (proj && 0 /* looks more correct without this */) + emit_mul(pc, t[2], t[2], src[3]); + else + if (arg == 4) /* there is no textureProj(samplerCubeShadow) */ + emit_mov(pc, t[3], src[3]); + + emit_flop(pc, NV50_FLOP_RCP, t[2], t[2]); + + emit_mul(pc, t[0], src[0], t[2]); + emit_mul(pc, t[1], src[1], t[2]); + emit_mul(pc, t[2], src[2], t[2]); +} + +static void +load_proj_tex_coords(struct nv50_pc *pc, struct nv50_reg *t[4], + struct nv50_reg **src, unsigned dim, unsigned arg) +{ + unsigned c, mode; + + if (src[0]->type == P_TEMP && src[0]->rhw != -1) { + mode = pc->interp_mode[src[0]->index] | INTERP_PERSPECTIVE; + + t[3]->rhw = src[3]->rhw; + emit_interp(pc, t[3], NULL, (mode & INTERP_CENTROID)); + emit_flop(pc, NV50_FLOP_RCP, t[3], t[3]); - unsigned c, mode, dim; + for (c = 0; c < dim; ++c) { + t[c]->rhw = src[c]->rhw; + emit_interp(pc, t[c], t[3], mode); + } + if (arg != dim) { /* depth reference value */ + t[dim]->rhw = src[2]->rhw; + emit_interp(pc, t[dim], t[3], mode); + } + } else { + /* XXX: for some reason the blob sometimes uses MAD + * (mad f32 $rX $rY $rZ neg $r63) + */ + emit_flop(pc, NV50_FLOP_RCP, t[3], src[3]); + for (c = 0; c < dim; ++c) + emit_mul(pc, t[c], src[c], t[3]); + if (arg != dim) /* depth reference value */ + emit_mul(pc, t[dim], src[2], t[3]); + } +} +static INLINE void +get_tex_dim(unsigned type, unsigned *dim, unsigned *arg) +{ switch (type) { case TGSI_TEXTURE_1D: - dim = 1; + *arg = *dim = 1; + break; + case TGSI_TEXTURE_SHADOW1D: + *dim = 1; + *arg = 2; break; case TGSI_TEXTURE_UNKNOWN: case TGSI_TEXTURE_2D: - case TGSI_TEXTURE_SHADOW1D: /* XXX: x, z */ case TGSI_TEXTURE_RECT: - dim = 2; + *arg = *dim = 2; + break; + case TGSI_TEXTURE_SHADOW2D: + case TGSI_TEXTURE_SHADOWRECT: + *dim = 2; + *arg = 3; break; case TGSI_TEXTURE_3D: case TGSI_TEXTURE_CUBE: - case TGSI_TEXTURE_SHADOW2D: - case TGSI_TEXTURE_SHADOWRECT: /* XXX */ - dim = 3; + *dim = *arg = 3; break; default: assert(0); break; } +} - /* some cards need t[0]'s hw index to be a multiple of 4 */ - alloc_temp4(pc, t, 0); +/* We shouldn't execute TEXLOD if any of the pixels in a quad have + * different LOD values, so branch off groups of equal LOD. + */ +static void +emit_texlod_sequence(struct nv50_pc *pc, struct nv50_reg *tlod, + struct nv50_reg *src, struct nv50_program_exec *tex) +{ + struct nv50_program_exec *join_at; + unsigned i, target = pc->p->exec_size + 9 * 2; - if (proj) { - if (src[0]->type == P_TEMP && src[0]->rhw != -1) { - mode = pc->interp_mode[src[0]->index]; + if (pc->p->type != PIPE_SHADER_FRAGMENT) { + emit(pc, tex); + return; + } + pc->allow32 = FALSE; - t[3]->rhw = src[3]->rhw; - emit_interp(pc, t[3], NULL, (mode & INTERP_CENTROID)); - emit_flop(pc, 0, t[3], t[3]); + /* Subtract lod of each pixel from lod of top left pixel, jump + * texlod insn if result is 0, then repeat for 2 other pixels. + */ + join_at = emit_joinat(pc); + emit_quadop(pc, NULL, 0, 0, tlod, tlod, 0x55); + emit_branch(pc, 0, 2)->param.index = target; - for (c = 0; c < dim; c++) { - t[c]->rhw = src[c]->rhw; - emit_interp(pc, t[c], t[3], - (mode | INTERP_PERSPECTIVE)); - } - } else { - emit_flop(pc, 0, t[3], src[3]); - for (c = 0; c < dim; c++) - emit_mul(pc, t[c], src[c], t[3]); + for (i = 1; i < 4; ++i) { + emit_quadop(pc, NULL, 0, i, tlod, tlod, 0x55); + emit_branch(pc, 0, 2)->param.index = target; + } - /* XXX: for some reason the blob sometimes uses MAD: - * emit_mad(pc, t[c], src[0][c], t[3], t[3]) - * pc->p->exec_tail->inst[1] |= 0x080fc000; - */ - } - } else { - if (type == TGSI_TEXTURE_CUBE) { - temp = temp_temp(pc); - emit_minmax(pc, 4, temp, src[0], src[1]); - emit_minmax(pc, 4, temp, temp, src[2]); - emit_flop(pc, 0, temp, temp); - for (c = 0; c < 3; c++) - emit_mul(pc, t[c], src[c], temp); - } else { - for (c = 0; c < dim; c++) - emit_mov(pc, t[c], src[c]); + emit_mov(pc, tlod, src); /* target */ + emit(pc, tex); /* texlod */ + + join_at->param.index = target + 2 * 2; + JOIN_ON(emit_nop(pc)); /* join _after_ tex */ +} + +static void +emit_texbias_sequence(struct nv50_pc *pc, struct nv50_reg *t[4], unsigned arg, + struct nv50_program_exec *tex) +{ + struct nv50_program_exec *e; + struct nv50_reg imm_1248, *t123[4][4], *r_bits = alloc_temp(pc, NULL); + int r_pred = 0; + unsigned n, c, i, cc[4] = { 0x0a, 0x13, 0x11, 0x10 }; + + pc->allow32 = FALSE; + ctor_reg(&imm_1248, P_IMMD, -1, ctor_immd_4u32(pc, 1, 2, 4, 8) * 4); + + /* Subtract bias value of thread i from bias values of each thread, + * store result in r_pred, and set bit i in r_bits if result was 0. + */ + assert(arg < 4); + for (i = 0; i < 4; ++i, ++imm_1248.hw) { + emit_quadop(pc, NULL, r_pred, i, t[arg], t[arg], 0x55); + emit_mov(pc, r_bits, &imm_1248); + set_pred(pc, 2, r_pred, pc->p->exec_tail); + } + emit_mov_to_pred(pc, r_pred, r_bits); + + /* The lanes of a quad are now grouped by the bit in r_pred they have + * set. Put the input values for TEX into a new register set for each + * group and execute TEX only for a specific group. + * We cannot use the same register set for each group because we need + * the derivatives, which are implicitly calculated, to be correct. + */ + for (i = 1; i < 4; ++i) { + alloc_temp4(pc, t123[i], 0); + + for (c = 0; c <= arg; ++c) + emit_mov(pc, t123[i][c], t[c]); + + *(e = exec(pc)) = *(tex); + e->inst[0] &= ~0x01fc; + set_dst(pc, t123[i][0], e); + set_pred(pc, cc[i], r_pred, e); + emit(pc, e); + } + /* finally TEX on the original regs (where we kept the input) */ + set_pred(pc, cc[0], r_pred, tex); + emit(pc, tex); + + /* put the 3 * n other results into regs for lane 0 */ + n = popcnt4(((e->inst[0] >> 25) & 0x3) | ((e->inst[1] >> 12) & 0xc)); + for (i = 1; i < 4; ++i) { + for (c = 0; c < n; ++c) { + emit_mov(pc, t[c], t123[i][c]); + set_pred(pc, cc[i], r_pred, pc->p->exec_tail); } + free_temp4(pc, t123[i]); } + emit_nop(pc); + free_temp(pc, r_bits); +} + +static void +emit_tex(struct nv50_pc *pc, struct nv50_reg **dst, unsigned mask, + struct nv50_reg **src, unsigned unit, unsigned type, + boolean proj, int bias_lod) +{ + struct nv50_reg *t[4]; + struct nv50_program_exec *e; + unsigned c, dim, arg; + + /* t[i] must be within a single 128 bit super-reg */ + alloc_temp4(pc, t, 0); + e = exec(pc); + e->inst[0] = 0xf0000000; set_long(pc, e); - e->inst[0] |= 0xf0000000; - e->inst[1] |= 0x00000004; set_dst(pc, t[0], e); - e->inst[0] |= (unit << 9); - if (dim == 2) - e->inst[0] |= 0x00400000; - else - if (dim == 3) - e->inst[0] |= 0x00800000; + /* TIC and TSC binding indices (TSC is ignored as TSC_LINKED = TRUE): */ + e->inst[0] |= (unit << 9) /* | (unit << 17) */; + + /* live flag (don't set if TEX results affect input to another TEX): */ + /* e->inst[0] |= 0x00000004; */ + + get_tex_dim(type, &dim, &arg); + + if (type == TGSI_TEXTURE_CUBE) { + e->inst[0] |= 0x08000000; + load_cube_tex_coords(pc, t, src, arg, proj); + } else + if (proj) + load_proj_tex_coords(pc, t, src, dim, arg); + else { + for (c = 0; c < dim; c++) + emit_mov(pc, t[c], src[c]); + if (arg != dim) /* depth reference value (always src.z here) */ + emit_mov(pc, t[dim], src[2]); + } e->inst[0] |= (mask & 0x3) << 25; e->inst[1] |= (mask & 0xc) << 12; - emit(pc, e); + if (!bias_lod) { + e->inst[0] |= (arg - 1) << 22; + emit(pc, e); + } else + if (bias_lod < 0) { + assert(pc->p->type == PIPE_SHADER_FRAGMENT); + e->inst[0] |= arg << 22; + e->inst[1] |= 0x20000000; /* texbias */ + emit_mov(pc, t[arg], src[3]); + emit_texbias_sequence(pc, t, arg, e); + } else { + e->inst[0] |= arg << 22; + e->inst[1] |= 0x40000000; /* texlod */ + emit_mov(pc, t[arg], src[3]); + emit_texlod_sequence(pc, t[arg], src[3], e); + } #if 1 c = 0; @@ -1187,34 +1829,36 @@ emit_tex(struct nv50_pc *pc, struct nv50_reg **dst, unsigned mask, } static void -emit_branch(struct nv50_pc *pc, int pred, unsigned cc, - struct nv50_program_exec **join) +emit_ddx(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) { struct nv50_program_exec *e = exec(pc); - if (join) { - set_long(pc, e); - e->inst[0] |= 0xa0000002; - emit(pc, e); - *join = e; - e = exec(pc); - } + assert(src->type == P_TEMP); + e->inst[0] = (src->mod & NV50_MOD_NEG) ? 0xc0240000 : 0xc0140000; + e->inst[1] = (src->mod & NV50_MOD_NEG) ? 0x86400000 : 0x89800000; set_long(pc, e); - e->inst[0] |= 0x10000002; - if (pred >= 0) - set_pred(pc, cc, pred, e); + set_dst(pc, dst, e); + set_src_0(pc, src, e); + set_src_2(pc, src, e); + emit(pc, e); } static void -emit_nop(struct nv50_pc *pc) +emit_ddy(struct nv50_pc *pc, struct nv50_reg *dst, struct nv50_reg *src) { struct nv50_program_exec *e = exec(pc); - e->inst[0] = 0xf0000000; + assert(src->type == P_TEMP); + + e->inst[0] = (src->mod & NV50_MOD_NEG) ? 0xc0250000 : 0xc0150000; + e->inst[1] = (src->mod & NV50_MOD_NEG) ? 0x85800000 : 0x8a400000; set_long(pc, e); - e->inst[1] = 0xe0000000; + set_dst(pc, dst, e); + set_src_0(pc, src, e); + set_src_2(pc, src, e); + emit(pc, e); } @@ -1231,6 +1875,17 @@ convert_to_long(struct nv50_pc *pc, struct nv50_program_exec *e) q = 0x0403c000; m = 0xffff7fff; break; + case 0x2: + case 0x3: + /* ADD, SUB, SUBR b32 */ + m = ~(0x8000 | (127 << 16)); + q = ((e->inst[0] & (~m)) >> 2) | (1 << 26); + break; + case 0x5: + /* SAD */ + m = ~(0x81 << 8); + q = 0x0c << 24; + break; case 0x8: /* INTERP (move centroid, perspective and flat bits) */ m = ~0x03000100; @@ -1266,22 +1921,55 @@ convert_to_long(struct nv50_pc *pc, struct nv50_program_exec *e) e->inst[1] |= q; } -static boolean -negate_supported(const struct tgsi_full_instruction *insn, int i) +/* Some operations support an optional negation flag. */ +static int +get_supported_mods(const struct tgsi_full_instruction *insn, int i) { switch (insn->Instruction.Opcode) { + case TGSI_OPCODE_ADD: + case TGSI_OPCODE_COS: + case TGSI_OPCODE_DDX: + case TGSI_OPCODE_DDY: case TGSI_OPCODE_DP3: case TGSI_OPCODE_DP4: - case TGSI_OPCODE_MUL: + case TGSI_OPCODE_EX2: case TGSI_OPCODE_KIL: - case TGSI_OPCODE_ADD: - case TGSI_OPCODE_SUB: + case TGSI_OPCODE_LG2: case TGSI_OPCODE_MAD: - return TRUE; + case TGSI_OPCODE_MUL: case TGSI_OPCODE_POW: - return (i == 1) ? TRUE : FALSE; + case TGSI_OPCODE_RCP: + case TGSI_OPCODE_RSQ: /* ignored, RSQ = rsqrt(abs(src.x)) */ + case TGSI_OPCODE_SCS: + case TGSI_OPCODE_SIN: + case TGSI_OPCODE_SUB: + return NV50_MOD_NEG; + case TGSI_OPCODE_MAX: + case TGSI_OPCODE_MIN: + case TGSI_OPCODE_INEG: /* tgsi src sign toggle/set would be stupid */ + return NV50_MOD_ABS; + case TGSI_OPCODE_CEIL: + case TGSI_OPCODE_FLR: + case TGSI_OPCODE_TRUNC: + return NV50_MOD_NEG | NV50_MOD_ABS; + case TGSI_OPCODE_F2I: + case TGSI_OPCODE_F2U: + case TGSI_OPCODE_I2F: + case TGSI_OPCODE_U2F: + return NV50_MOD_NEG | NV50_MOD_ABS | NV50_MOD_I32; + case TGSI_OPCODE_UADD: + return NV50_MOD_NEG | NV50_MOD_I32; + case TGSI_OPCODE_SAD: + case TGSI_OPCODE_SHL: + case TGSI_OPCODE_IMAX: + case TGSI_OPCODE_IMIN: + case TGSI_OPCODE_ISHR: + case TGSI_OPCODE_UMAX: + case TGSI_OPCODE_UMIN: + case TGSI_OPCODE_USHR: + return NV50_MOD_I32; default: - return FALSE; + return 0; } } @@ -1289,7 +1977,7 @@ negate_supported(const struct tgsi_full_instruction *insn, int i) static unsigned nv50_tgsi_src_mask(const struct tgsi_full_instruction *insn, int c) { - unsigned x, mask = insn->FullDstRegisters[0].DstRegister.WriteMask; + unsigned x, mask = insn->Dst[0].Register.WriteMask; switch (insn->Instruction.Opcode) { case TGSI_OPCODE_COS: @@ -1304,30 +1992,40 @@ nv50_tgsi_src_mask(const struct tgsi_full_instruction *insn, int c) case TGSI_OPCODE_DST: return mask & (c ? 0xa : 0x6); case TGSI_OPCODE_EX2: + case TGSI_OPCODE_EXP: case TGSI_OPCODE_LG2: + case TGSI_OPCODE_LOG: case TGSI_OPCODE_POW: case TGSI_OPCODE_RCP: case TGSI_OPCODE_RSQ: case TGSI_OPCODE_SCS: return 0x1; + case TGSI_OPCODE_IF: + return 0x1; case TGSI_OPCODE_LIT: return 0xb; case TGSI_OPCODE_TEX: + case TGSI_OPCODE_TXB: + case TGSI_OPCODE_TXL: case TGSI_OPCODE_TXP: { - const struct tgsi_instruction_ext_texture *tex; + const struct tgsi_instruction_texture *tex; - assert(insn->Instruction.Extended); - tex = &insn->InstructionExtTexture; + assert(insn->Instruction.Texture); + tex = &insn->Texture; mask = 0x7; - if (insn->Instruction.Opcode == TGSI_OPCODE_TXP) - mask |= 0x8; + if (insn->Instruction.Opcode != TGSI_OPCODE_TEX && + insn->Instruction.Opcode != TGSI_OPCODE_TXD) + mask |= 0x8; /* bias, lod or proj */ switch (tex->Texture) { case TGSI_TEXTURE_1D: mask &= 0x9; break; + case TGSI_TEXTURE_SHADOW1D: + mask &= 0x5; + break; case TGSI_TEXTURE_2D: mask &= 0xb; break; @@ -1352,11 +2050,21 @@ nv50_tgsi_src_mask(const struct tgsi_full_instruction *insn, int c) static struct nv50_reg * tgsi_dst(struct nv50_pc *pc, int c, const struct tgsi_full_dst_register *dst) { - switch (dst->DstRegister.File) { + switch (dst->Register.File) { case TGSI_FILE_TEMPORARY: - return &pc->temp[dst->DstRegister.Index * 4 + c]; + return &pc->temp[dst->Register.Index * 4 + c]; case TGSI_FILE_OUTPUT: - return &pc->result[dst->DstRegister.Index * 4 + c]; + return &pc->result[dst->Register.Index * 4 + c]; + case TGSI_FILE_ADDRESS: + { + struct nv50_reg *r = pc->addr[dst->Register.Index * 4 + c]; + if (!r) { + r = alloc_addr(pc, NULL); + pc->addr[dst->Register.Index * 4 + c] = r; + } + assert(r); + return r; + } case TGSI_FILE_NULL: return NULL; default: @@ -1368,83 +2076,93 @@ tgsi_dst(struct nv50_pc *pc, int c, const struct tgsi_full_dst_register *dst) static struct nv50_reg * tgsi_src(struct nv50_pc *pc, int chan, const struct tgsi_full_src_register *src, - boolean neg) + int mod) { struct nv50_reg *r = NULL; - struct nv50_reg *temp; - unsigned sgn, c; + struct nv50_reg *temp = NULL; + unsigned sgn, c, swz, cvn; + + if (src->Register.File != TGSI_FILE_CONSTANT) + assert(!src->Register.Indirect); sgn = tgsi_util_get_full_src_register_sign_mode(src, chan); - c = tgsi_util_get_full_src_register_extswizzle(src, chan); + c = tgsi_util_get_full_src_register_swizzle(src, chan); switch (c) { - case TGSI_EXTSWIZZLE_X: - case TGSI_EXTSWIZZLE_Y: - case TGSI_EXTSWIZZLE_Z: - case TGSI_EXTSWIZZLE_W: - switch (src->SrcRegister.File) { + case TGSI_SWIZZLE_X: + case TGSI_SWIZZLE_Y: + case TGSI_SWIZZLE_Z: + case TGSI_SWIZZLE_W: + switch (src->Register.File) { case TGSI_FILE_INPUT: - r = &pc->attr[src->SrcRegister.Index * 4 + c]; + r = &pc->attr[src->Register.Index * 4 + c]; break; case TGSI_FILE_TEMPORARY: - r = &pc->temp[src->SrcRegister.Index * 4 + c]; + r = &pc->temp[src->Register.Index * 4 + c]; break; case TGSI_FILE_CONSTANT: - r = &pc->param[src->SrcRegister.Index * 4 + c]; + if (!src->Register.Indirect) { + r = &pc->param[src->Register.Index * 4 + c]; + break; + } + /* Indicate indirection by setting r->acc < 0 and + * use the index field to select the address reg. + */ + r = reg_instance(pc, NULL); + swz = tgsi_util_get_src_register_swizzle( + &src->Indirect, 0); + ctor_reg(r, P_CONST, + src->Indirect.Index * 4 + swz, + src->Register.Index * 4 + c); + r->acc = -1; break; case TGSI_FILE_IMMEDIATE: - r = &pc->immd[src->SrcRegister.Index * 4 + c]; + r = &pc->immd[src->Register.Index * 4 + c]; break; case TGSI_FILE_SAMPLER: + return NULL; + case TGSI_FILE_ADDRESS: + r = pc->addr[src->Register.Index * 4 + c]; + assert(r); break; default: assert(0); break; } break; - case TGSI_EXTSWIZZLE_ZERO: - r = alloc_immd(pc, 0.0); - return r; - case TGSI_EXTSWIZZLE_ONE: - if (sgn == TGSI_UTIL_SIGN_TOGGLE || sgn == TGSI_UTIL_SIGN_SET) - return alloc_immd(pc, -1.0); - return alloc_immd(pc, 1.0); default: assert(0); break; } + cvn = (mod & NV50_MOD_I32) ? CVT_S32_S32 : CVT_F32_F32; + switch (sgn) { - case TGSI_UTIL_SIGN_KEEP: - break; case TGSI_UTIL_SIGN_CLEAR: - temp = temp_temp(pc); - emit_abs(pc, temp, r); - r = temp; - break; - case TGSI_UTIL_SIGN_TOGGLE: - if (neg) - r->neg = 1; - else { - temp = temp_temp(pc); - emit_neg(pc, temp, r); - r = temp; - } + r->mod = NV50_MOD_ABS; break; case TGSI_UTIL_SIGN_SET: - temp = temp_temp(pc); - emit_abs(pc, temp, r); - if (neg) - temp->neg = 1; - else - emit_neg(pc, temp, temp); - r = temp; + r->mod = NV50_MOD_NEG_ABS; + break; + case TGSI_UTIL_SIGN_TOGGLE: + r->mod = NV50_MOD_NEG; break; default: - assert(0); + assert(!r->mod && sgn == TGSI_UTIL_SIGN_KEEP); break; } + if ((r->mod & mod) != r->mod) { + temp = temp_temp(pc); + emit_cvt(pc, temp, r, -1, cvn); + r->mod = 0; + r = temp; + } else + r->mod |= mod & NV50_MOD_I32; + + assert(r); + if (r->acc >= 0 && r != temp) + return reg_instance(pc, r); /* will clear r->mod */ return r; } @@ -1497,9 +2215,13 @@ nv50_tgsi_dst_revdep(unsigned op, int s, int c) assert(0); return 0x0; } + case TGSI_OPCODE_EXP: + case TGSI_OPCODE_LOG: case TGSI_OPCODE_LIT: case TGSI_OPCODE_SCS: case TGSI_OPCODE_TEX: + case TGSI_OPCODE_TXB: + case TGSI_OPCODE_TXL: case TGSI_OPCODE_TXP: /* these take care of dangerous swizzles themselves */ return 0x0; @@ -1535,34 +2257,51 @@ nv50_kill_branch(struct nv50_pc *pc) if (pc->if_insn[lvl]->next != pc->p->exec_tail) return FALSE; + if (is_immd(pc->p->exec_tail)) + return FALSE; /* if ccode == 'true', the BRA is from an ELSE and the predicate * reg may no longer be valid, since we currently always use $p0 */ if (has_pred(pc->if_insn[lvl], 0xf)) return FALSE; - assert(pc->if_insn[lvl] && pc->br_join[lvl]); + assert(pc->if_insn[lvl] && pc->if_join[lvl]); - /* We'll use the exec allocated for JOIN_AT (as we can't easily - * update prev's next); if exec_tail is BRK, update the pointer. + /* We'll use the exec allocated for JOIN_AT (we can't easily + * access nv50_program_exec's prev). */ - if (pc->loop_lvl && pc->br_loop[pc->loop_lvl - 1] == pc->p->exec_tail) - pc->br_loop[pc->loop_lvl - 1] = pc->br_join[lvl]; - pc->p->exec_size -= 4; /* remove JOIN_AT and BRA */ - *pc->br_join[lvl] = *pc->p->exec_tail; + *pc->if_join[lvl] = *pc->p->exec_tail; FREE(pc->if_insn[lvl]); FREE(pc->p->exec_tail); - pc->p->exec_tail = pc->br_join[lvl]; + pc->p->exec_tail = pc->if_join[lvl]; pc->p->exec_tail->next = NULL; set_pred(pc, 0xd, 0, pc->p->exec_tail); return TRUE; } +static void +nv50_fp_move_results(struct nv50_pc *pc) +{ + struct nv50_reg reg; + unsigned i; + + ctor_reg(®, P_TEMP, -1, -1); + + for (i = 0; i < pc->result_nr * 4; ++i) { + if (pc->result[i].rhw < 0 || pc->result[i].hw < 0) + continue; + if (pc->result[i].rhw != pc->result[i].hw) { + reg.hw = pc->result[i].rhw; + emit_mov(pc, ®, &pc->result[i]); + } + } +} + static boolean nv50_program_tx_insn(struct nv50_pc *pc, const struct tgsi_full_instruction *inst) @@ -1571,33 +2310,33 @@ nv50_program_tx_insn(struct nv50_pc *pc, unsigned mask, sat, unit; int i, c; - mask = inst->FullDstRegisters[0].DstRegister.WriteMask; + mask = inst->Dst[0].Register.WriteMask; sat = inst->Instruction.Saturate == TGSI_SAT_ZERO_ONE; memset(src, 0, sizeof(src)); for (c = 0; c < 4; c++) { if ((mask & (1 << c)) && !pc->r_dst[c]) - dst[c] = tgsi_dst(pc, c, &inst->FullDstRegisters[0]); + dst[c] = tgsi_dst(pc, c, &inst->Dst[0]); else dst[c] = pc->r_dst[c]; rdst[c] = dst[c]; } for (i = 0; i < inst->Instruction.NumSrcRegs; i++) { - const struct tgsi_full_src_register *fs = &inst->FullSrcRegisters[i]; + const struct tgsi_full_src_register *fs = &inst->Src[i]; unsigned src_mask; - boolean neg_supp; + int mod_supp; src_mask = nv50_tgsi_src_mask(inst, i); - neg_supp = negate_supported(inst, i); + mod_supp = get_supported_mods(inst, i); - if (fs->SrcRegister.File == TGSI_FILE_SAMPLER) - unit = fs->SrcRegister.Index; + if (fs->Register.File == TGSI_FILE_SAMPLER) + unit = fs->Register.Index; for (c = 0; c < 4; c++) if (src_mask & (1 << c)) - src[i][c] = tgsi_src(pc, c, fs, neg_supp); + src[i][c] = tgsi_src(pc, c, fs, mod_supp); } brdc = temp = pc->r_brdc; @@ -1610,7 +2349,7 @@ nv50_program_tx_insn(struct nv50_pc *pc, for (c = 0; c < 4; c++) { if (!(mask & (1 << c)) || dst[c]->type == P_TEMP) continue; - rdst[c] = dst[c]; + /* rdst[c] = dst[c]; */ /* done above */ dst[c] = temp_temp(pc); } } @@ -1622,7 +2361,8 @@ nv50_program_tx_insn(struct nv50_pc *pc, for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - emit_abs(pc, dst[c], src[0][c]); + emit_cvt(pc, dst[c], src[0][c], -1, + CVT_ABS | CVT_F32_F32); } break; case TGSI_OPCODE_ADD: @@ -1632,33 +2372,93 @@ nv50_program_tx_insn(struct nv50_pc *pc, emit_add(pc, dst[c], src[0][c], src[1][c]); } break; + case TGSI_OPCODE_AND: + case TGSI_OPCODE_XOR: + case TGSI_OPCODE_OR: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_bitop2(pc, dst[c], src[0][c], src[1][c], + inst->Instruction.Opcode); + } + break; + case TGSI_OPCODE_ARL: + assert(src[0][0]); + temp = temp_temp(pc); + emit_cvt(pc, temp, src[0][0], -1, CVT_FLOOR | CVT_S32_F32); + emit_arl(pc, dst[0], temp, 4); + break; case TGSI_OPCODE_BGNLOOP: + pc->loop_brka[pc->loop_lvl] = emit_breakaddr(pc); pc->loop_pos[pc->loop_lvl++] = pc->p->exec_size; + terminate_mbb(pc); + break; + case TGSI_OPCODE_BGNSUB: + assert(!pc->in_subroutine); + pc->in_subroutine = TRUE; + /* probably not necessary, but align to 8 byte boundary */ + if (!is_long(pc->p->exec_tail)) + convert_to_long(pc, pc->p->exec_tail); break; case TGSI_OPCODE_BRK: - emit_branch(pc, -1, 0, NULL); assert(pc->loop_lvl > 0); - pc->br_loop[pc->loop_lvl - 1] = pc->p->exec_tail; + emit_break(pc, -1, 0); + break; + case TGSI_OPCODE_CAL: + assert(inst->Label.Label < pc->insn_nr); + emit_call(pc, -1, 0)->param.index = inst->Label.Label; + /* replaced by actual offset in nv50_program_fixup_insns */ break; case TGSI_OPCODE_CEIL: for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; emit_cvt(pc, dst[c], src[0][c], -1, - CVTOP_CEIL, CVT_F32_F32 | CVT_RI); + CVT_CEIL | CVT_F32_F32 | CVT_RI); } break; + case TGSI_OPCODE_CMP: + pc->allow32 = FALSE; + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, NULL, src[0][c], 1, CVT_F32_F32); + emit_mov(pc, dst[c], src[1][c]); + set_pred(pc, 0x1, 1, pc->p->exec_tail); /* @SF */ + emit_mov(pc, dst[c], src[2][c]); + set_pred(pc, 0x6, 1, pc->p->exec_tail); /* @NSF */ + } + break; + case TGSI_OPCODE_CONT: + assert(pc->loop_lvl > 0); + emit_branch(pc, -1, 0)->param.index = + pc->loop_pos[pc->loop_lvl - 1]; + break; case TGSI_OPCODE_COS: if (mask & 8) { emit_precossin(pc, temp, src[0][3]); - emit_flop(pc, 5, dst[3], temp); + emit_flop(pc, NV50_FLOP_COS, dst[3], temp); if (!(mask &= 7)) break; if (temp == dst[3]) temp = brdc = temp_temp(pc); } emit_precossin(pc, temp, src[0][0]); - emit_flop(pc, 5, brdc, temp); + emit_flop(pc, NV50_FLOP_COS, brdc, temp); + break; + case TGSI_OPCODE_DDX: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_ddx(pc, dst[c], src[0][c]); + } + break; + case TGSI_OPCODE_DDY: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_ddy(pc, dst[c], src[0][c]); + } break; case TGSI_OPCODE_DP3: emit_mul(pc, temp, src[0][0], src[1][0]); @@ -1688,9 +2488,10 @@ nv50_program_tx_insn(struct nv50_pc *pc, emit_mov_immdval(pc, dst[0], 1.0f); break; case TGSI_OPCODE_ELSE: - emit_branch(pc, -1, 0, NULL); + emit_branch(pc, -1, 0); pc->if_insn[--pc->if_lvl]->param.index = pc->p->exec_size; pc->if_insn[pc->if_lvl++] = pc->p->exec_tail; + terminate_mbb(pc); break; case TGSI_OPCODE_ENDIF: pc->if_insn[--pc->if_lvl]->param.index = pc->p->exec_size; @@ -1699,24 +2500,72 @@ nv50_program_tx_insn(struct nv50_pc *pc, if (nv50_kill_branch(pc) == TRUE) break; - if (pc->br_join[pc->if_lvl]) { - pc->br_join[pc->if_lvl]->param.index = pc->p->exec_size; - pc->br_join[pc->if_lvl] = NULL; + if (pc->if_join[pc->if_lvl]) { + pc->if_join[pc->if_lvl]->param.index = pc->p->exec_size; + pc->if_join[pc->if_lvl] = NULL; } + terminate_mbb(pc); /* emit a NOP as join point, we could set it on the next * one, but would have to make sure it is long and !immd */ - emit_nop(pc); - pc->p->exec_tail->inst[1] |= 2; + JOIN_ON(emit_nop(pc)); break; case TGSI_OPCODE_ENDLOOP: - emit_branch(pc, -1, 0, NULL); - pc->p->exec_tail->param.index = pc->loop_pos[--pc->loop_lvl]; - pc->br_loop[pc->loop_lvl]->param.index = pc->p->exec_size; + emit_branch(pc, -1, 0)->param.index = + pc->loop_pos[--pc->loop_lvl]; + pc->loop_brka[pc->loop_lvl]->param.index = pc->p->exec_size; + terminate_mbb(pc); + break; + case TGSI_OPCODE_ENDSUB: + assert(pc->in_subroutine); + pc->in_subroutine = FALSE; break; case TGSI_OPCODE_EX2: emit_preex2(pc, temp, src[0][0]); - emit_flop(pc, 6, brdc, temp); + emit_flop(pc, NV50_FLOP_EX2, brdc, temp); + break; + case TGSI_OPCODE_EXP: + { + struct nv50_reg *t[2]; + + assert(!temp); + t[0] = temp_temp(pc); + t[1] = temp_temp(pc); + + if (mask & 0x6) + emit_mov(pc, t[0], src[0][0]); + if (mask & 0x3) + emit_flr(pc, t[1], src[0][0]); + + if (mask & (1 << 1)) + emit_sub(pc, dst[1], t[0], t[1]); + if (mask & (1 << 0)) { + emit_preex2(pc, t[1], t[1]); + emit_flop(pc, NV50_FLOP_EX2, dst[0], t[1]); + } + if (mask & (1 << 2)) { + emit_preex2(pc, t[0], t[0]); + emit_flop(pc, NV50_FLOP_EX2, dst[2], t[0]); + } + if (mask & (1 << 3)) + emit_mov_immdval(pc, dst[3], 1.0f); + } + break; + case TGSI_OPCODE_F2I: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, dst[c], src[0][c], -1, + CVT_TRUNC | CVT_S32_F32); + } + break; + case TGSI_OPCODE_F2U: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, dst[c], src[0][c], -1, + CVT_TRUNC | CVT_U32_F32); + } break; case TGSI_OPCODE_FLR: for (c = 0; c < 4; c++) { @@ -1734,24 +2583,85 @@ nv50_program_tx_insn(struct nv50_pc *pc, emit_sub(pc, dst[c], src[0][c], temp); } break; + case TGSI_OPCODE_I2F: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, dst[c], src[0][c], -1, CVT_F32_S32); + } + break; case TGSI_OPCODE_IF: - /* emitting a join_at may not be necessary */ - assert(pc->if_lvl < MAX_IF_DEPTH); - set_pred_wr(pc, 1, 0, pc->if_cond); - emit_branch(pc, 0, 2, &pc->br_join[pc->if_lvl]); - pc->if_insn[pc->if_lvl++] = pc->p->exec_tail; + assert(pc->if_lvl < NV50_MAX_COND_NESTING); + emit_cvt(pc, NULL, src[0][0], 0, CVT_ABS | CVT_F32_F32); + pc->if_join[pc->if_lvl] = emit_joinat(pc); + pc->if_insn[pc->if_lvl++] = emit_branch(pc, 0, 2);; + terminate_mbb(pc); + break; + case TGSI_OPCODE_IMAX: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_minmax(pc, 0x08c, dst[c], src[0][c], src[1][c]); + } + break; + case TGSI_OPCODE_IMIN: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_minmax(pc, 0x0ac, dst[c], src[0][c], src[1][c]); + } + break; + case TGSI_OPCODE_INEG: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, dst[c], src[0][c], -1, + CVT_S32_S32 | CVT_NEG); + } break; case TGSI_OPCODE_KIL: + assert(src[0][0] && src[0][1] && src[0][2] && src[0][3]); emit_kil(pc, src[0][0]); emit_kil(pc, src[0][1]); emit_kil(pc, src[0][2]); emit_kil(pc, src[0][3]); break; + case TGSI_OPCODE_KILP: + emit_kil(pc, NULL); + break; case TGSI_OPCODE_LIT: emit_lit(pc, &dst[0], mask, &src[0][0]); break; case TGSI_OPCODE_LG2: - emit_flop(pc, 3, brdc, src[0][0]); + emit_flop(pc, NV50_FLOP_LG2, brdc, src[0][0]); + break; + case TGSI_OPCODE_LOG: + { + struct nv50_reg *t[2]; + + t[0] = temp_temp(pc); + if (mask & (1 << 1)) + t[1] = temp_temp(pc); + else + t[1] = t[0]; + + emit_cvt(pc, t[0], src[0][0], -1, CVT_ABS | CVT_F32_F32); + emit_flop(pc, NV50_FLOP_LG2, t[1], t[0]); + if (mask & (1 << 2)) + emit_mov(pc, dst[2], t[1]); + emit_flr(pc, t[1], t[1]); + if (mask & (1 << 0)) + emit_mov(pc, dst[0], t[1]); + if (mask & (1 << 1)) { + t[1]->mod = NV50_MOD_NEG; + emit_preex2(pc, t[1], t[1]); + t[1]->mod = 0; + emit_flop(pc, NV50_FLOP_EX2, t[1], t[1]); + emit_mul(pc, dst[1], t[0], t[1]); + } + if (mask & (1 << 3)) + emit_mov_immdval(pc, dst[3], 1.0f); + } break; case TGSI_OPCODE_LRP: temp = temp_temp(pc); @@ -1773,18 +2683,17 @@ nv50_program_tx_insn(struct nv50_pc *pc, for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - emit_minmax(pc, 4, dst[c], src[0][c], src[1][c]); + emit_minmax(pc, 0x880, dst[c], src[0][c], src[1][c]); } break; case TGSI_OPCODE_MIN: for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - emit_minmax(pc, 5, dst[c], src[0][c], src[1][c]); + emit_minmax(pc, 0x8a0, dst[c], src[0][c], src[1][c]); } break; case TGSI_OPCODE_MOV: - case TGSI_OPCODE_SWZ: for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; @@ -1802,35 +2711,58 @@ nv50_program_tx_insn(struct nv50_pc *pc, emit_pow(pc, brdc, src[0][0], src[1][0]); break; case TGSI_OPCODE_RCP: - emit_flop(pc, 0, brdc, src[0][0]); + emit_flop(pc, NV50_FLOP_RCP, brdc, src[0][0]); + break; + case TGSI_OPCODE_RET: + if (pc->p->type == PIPE_SHADER_FRAGMENT && !pc->in_subroutine) + nv50_fp_move_results(pc); + emit_ret(pc, -1, 0); break; case TGSI_OPCODE_RSQ: - emit_flop(pc, 2, brdc, src[0][0]); + src[0][0]->mod |= NV50_MOD_ABS; + emit_flop(pc, NV50_FLOP_RSQ, brdc, src[0][0]); + break; + case TGSI_OPCODE_SAD: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_sad(pc, dst[c], src[0][c], src[1][c], src[2][c]); + } break; case TGSI_OPCODE_SCS: temp = temp_temp(pc); if (mask & 3) emit_precossin(pc, temp, src[0][0]); if (mask & (1 << 0)) - emit_flop(pc, 5, dst[0], temp); + emit_flop(pc, NV50_FLOP_COS, dst[0], temp); if (mask & (1 << 1)) - emit_flop(pc, 4, dst[1], temp); + emit_flop(pc, NV50_FLOP_SIN, dst[1], temp); if (mask & (1 << 2)) emit_mov_immdval(pc, dst[2], 0.0); if (mask & (1 << 3)) emit_mov_immdval(pc, dst[3], 1.0); break; + case TGSI_OPCODE_SHL: + case TGSI_OPCODE_ISHR: + case TGSI_OPCODE_USHR: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_shift(pc, dst[c], src[0][c], src[1][c], + inst->Instruction.Opcode); + } + break; case TGSI_OPCODE_SIN: if (mask & 8) { emit_precossin(pc, temp, src[0][3]); - emit_flop(pc, 4, dst[3], temp); + emit_flop(pc, NV50_FLOP_SIN, dst[3], temp); if (!(mask &= 7)) break; if (temp == dst[3]) temp = brdc = temp_temp(pc); } emit_precossin(pc, temp, src[0][0]); - emit_flop(pc, 4, brdc, temp); + emit_flop(pc, NV50_FLOP_SIN, brdc, temp); break; case TGSI_OPCODE_SLT: case TGSI_OPCODE_SGE: @@ -1838,12 +2770,23 @@ nv50_program_tx_insn(struct nv50_pc *pc, case TGSI_OPCODE_SGT: case TGSI_OPCODE_SLE: case TGSI_OPCODE_SNE: - i = map_tgsi_setop_cc(inst->Instruction.Opcode); + case TGSI_OPCODE_ISLT: + case TGSI_OPCODE_ISGE: + case TGSI_OPCODE_USEQ: + case TGSI_OPCODE_USGE: + case TGSI_OPCODE_USLT: + case TGSI_OPCODE_USNE: + { + uint8_t cc, ty; + + map_tgsi_setop_hw(inst->Instruction.Opcode, &cc, &ty); + for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - emit_set(pc, i, dst[c], -1, src[0][c], src[1][c]); + emit_set(pc, cc, dst[c], -1, src[0][c], src[1][c], ty); } + } break; case TGSI_OPCODE_SUB: for (c = 0; c < 4; c++) { @@ -1854,18 +2797,54 @@ nv50_program_tx_insn(struct nv50_pc *pc, break; case TGSI_OPCODE_TEX: emit_tex(pc, dst, mask, src[0], unit, - inst->InstructionExtTexture.Texture, FALSE); + inst->Texture.Texture, FALSE, 0); + break; + case TGSI_OPCODE_TXB: + emit_tex(pc, dst, mask, src[0], unit, + inst->Texture.Texture, FALSE, -1); + break; + case TGSI_OPCODE_TXL: + emit_tex(pc, dst, mask, src[0], unit, + inst->Texture.Texture, FALSE, 1); break; case TGSI_OPCODE_TXP: emit_tex(pc, dst, mask, src[0], unit, - inst->InstructionExtTexture.Texture, TRUE); + inst->Texture.Texture, TRUE, 0); break; case TGSI_OPCODE_TRUNC: for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; emit_cvt(pc, dst[c], src[0][c], -1, - CVTOP_TRUNC, CVT_F32_F32 | CVT_RI); + CVT_TRUNC | CVT_F32_F32 | CVT_RI); + } + break; + case TGSI_OPCODE_U2F: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_cvt(pc, dst[c], src[0][c], -1, CVT_F32_U32); + } + break; + case TGSI_OPCODE_UADD: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_add_b32(pc, dst[c], src[0][c], src[1][c]); + } + break; + case TGSI_OPCODE_UMAX: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_minmax(pc, 0x084, dst[c], src[0][c], src[1][c]); + } + break; + case TGSI_OPCODE_UMIN: + for (c = 0; c < 4; c++) { + if (!(mask & (1 << c))) + continue; + emit_minmax(pc, 0x0a4, dst[c], src[0][c], src[1][c]); } break; case TGSI_OPCODE_XPD: @@ -1886,6 +2865,17 @@ nv50_program_tx_insn(struct nv50_pc *pc, emit_mov_immdval(pc, dst[3], 1.0); break; case TGSI_OPCODE_END: + if (pc->p->type == PIPE_SHADER_FRAGMENT) + nv50_fp_move_results(pc); + + /* last insn must be long so it can have the exit bit set */ + if (!is_long(pc->p->exec_tail)) + convert_to_long(pc, pc->p->exec_tail); + else + if (is_immd(pc->p->exec_tail) || is_join(pc->p->exec_tail)) + emit_nop(pc); + + pc->p->exec_tail->inst[1] |= 1; /* set exit bit */ break; default: NOUVEAU_ERR("invalid opcode %d\n", inst->Instruction.Opcode); @@ -1903,42 +2893,43 @@ nv50_program_tx_insn(struct nv50_pc *pc, for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - /* in this case we saturate later */ - if (dst[c]->type == P_TEMP && dst[c]->index < 0) + /* In this case we saturate later, and dst[c] won't + * be another temp_temp (and thus lost), since rdst + * already is TEMP (see above). */ + if (rdst[c]->type == P_TEMP && rdst[c]->index < 0) continue; emit_sat(pc, rdst[c], dst[c]); } } - for (i = 0; i < inst->Instruction.NumSrcRegs; i++) { - for (c = 0; c < 4; c++) { - if (!src[i][c]) - continue; - if (src[i][c]->index == -1 && src[i][c]->type == P_IMMD) - FREE(src[i][c]); - } - } - kill_temp_temp(pc); + pc->reg_instance_nr = 0; + return TRUE; } static void prep_inspect_insn(struct nv50_pc *pc, const struct tgsi_full_instruction *insn) { - struct nv50_reg *reg = NULL; + struct nv50_reg *r, *reg = NULL; const struct tgsi_full_src_register *src; const struct tgsi_dst_register *dst; unsigned i, c, k, mask; - dst = &insn->FullDstRegisters[0].DstRegister; + dst = &insn->Dst[0].Register; mask = dst->WriteMask; if (dst->File == TGSI_FILE_TEMPORARY) - reg = pc->temp; + reg = pc->temp; else - if (dst->File == TGSI_FILE_OUTPUT) - reg = pc->result; + if (dst->File == TGSI_FILE_OUTPUT) { + reg = pc->result; + + if (insn->Instruction.Opcode == TGSI_OPCODE_MOV && + dst->Index == pc->edgeflag_out && + insn->Src[0].Register.File == TGSI_FILE_INPUT) + pc->p->cfg.edgeflag_in = insn->Src[0].Register.Index; + } if (reg) { for (c = 0; c < 4; c++) { @@ -1949,12 +2940,12 @@ prep_inspect_insn(struct nv50_pc *pc, const struct tgsi_full_instruction *insn) } for (i = 0; i < insn->Instruction.NumSrcRegs; i++) { - src = &insn->FullSrcRegisters[i]; + src = &insn->Src[i]; - if (src->SrcRegister.File == TGSI_FILE_TEMPORARY) + if (src->Register.File == TGSI_FILE_TEMPORARY) reg = pc->temp; else - if (src->SrcRegister.File == TGSI_FILE_INPUT) + if (src->Register.File == TGSI_FILE_INPUT) reg = pc->attr; else continue; @@ -1964,12 +2955,17 @@ prep_inspect_insn(struct nv50_pc *pc, const struct tgsi_full_instruction *insn) for (c = 0; c < 4; c++) { if (!(mask & (1 << c))) continue; - k = tgsi_util_get_full_src_register_extswizzle(src, c); + k = tgsi_util_get_full_src_register_swizzle(src, c); - if (k > TGSI_EXTSWIZZLE_W) - continue; + r = ®[src->Register.Index * 4 + k]; + + /* If used before written, pre-allocate the reg, + * lest we overwrite results from a subroutine. + */ + if (!r->acc && r->type == P_TEMP) + alloc_reg(pc, r); - reg[src->SrcRegister.Index * 4 + k].acc = pc->insn_nr; + r->acc = pc->insn_nr; } } } @@ -2029,13 +3025,13 @@ static struct nv50_reg * tgsi_broadcast_dst(struct nv50_pc *pc, const struct tgsi_full_dst_register *fd, unsigned mask) { - if (fd->DstRegister.File == TGSI_FILE_TEMPORARY) { - int c = ffs(~mask & fd->DstRegister.WriteMask); + if (fd->Register.File == TGSI_FILE_TEMPORARY) { + int c = ffs(~mask & fd->Register.WriteMask); if (c) return tgsi_dst(pc, c - 1, fd); } else { - int c = ffs(fd->DstRegister.WriteMask) - 1; - if ((1 << c) == fd->DstRegister.WriteMask) + int c = ffs(fd->Register.WriteMask) - 1; + if ((1 << c) == fd->Register.WriteMask) return tgsi_dst(pc, c, fd); } @@ -2049,7 +3045,7 @@ static unsigned nv50_tgsi_scan_swizzle(const struct tgsi_full_instruction *insn, unsigned rdep[4]) { - const struct tgsi_full_dst_register *fd = &insn->FullDstRegisters[0]; + const struct tgsi_full_dst_register *fd = &insn->Dst[0]; const struct tgsi_full_src_register *fs; unsigned i, deqs = 0; @@ -2058,11 +3054,11 @@ nv50_tgsi_scan_swizzle(const struct tgsi_full_instruction *insn, for (i = 0; i < insn->Instruction.NumSrcRegs; i++) { unsigned chn, mask = nv50_tgsi_src_mask(insn, i); - boolean neg_supp = negate_supported(insn, i); + int ms = get_supported_mods(insn, i); - fs = &insn->FullSrcRegisters[i]; - if (fs->SrcRegister.File != fd->DstRegister.File || - fs->SrcRegister.Index != fd->DstRegister.Index) + fs = &insn->Src[i]; + if (fs->Register.File != fd->Register.File || + fs->Register.Index != fd->Register.Index) continue; for (chn = 0; chn < 4; ++chn) { @@ -2070,17 +3066,18 @@ nv50_tgsi_scan_swizzle(const struct tgsi_full_instruction *insn, if (!(mask & (1 << chn))) /* src is not read */ continue; - c = tgsi_util_get_full_src_register_extswizzle(fs, chn); + c = tgsi_util_get_full_src_register_swizzle(fs, chn); s = tgsi_util_get_full_src_register_sign_mode(fs, chn); - if (c > TGSI_EXTSWIZZLE_W || - !(fd->DstRegister.WriteMask & (1 << c))) + if (!(fd->Register.WriteMask & (1 << c))) continue; - /* no danger if src is copied to TEMP first */ - if ((s != TGSI_UTIL_SIGN_KEEP) && - (s != TGSI_UTIL_SIGN_TOGGLE || !neg_supp)) - continue; + if (s == TGSI_UTIL_SIGN_TOGGLE && !(ms & NV50_MOD_NEG)) + continue; + if (s == TGSI_UTIL_SIGN_CLEAR && !(ms & NV50_MOD_ABS)) + continue; + if ((s == TGSI_UTIL_SIGN_SET) && ((ms & 3) != 3)) + continue; rdep[c] |= nv50_tgsi_dst_revdep( insn->Instruction.Opcode, i, chn); @@ -2098,7 +3095,7 @@ nv50_tgsi_insn(struct nv50_pc *pc, const union tgsi_full_token *tok) const struct tgsi_full_dst_register *fd; unsigned i, deqs, rdep[4], m[4]; - fd = &tok->FullInstruction.FullDstRegisters[0]; + fd = &tok->FullInstruction.Dst[0]; deqs = nv50_tgsi_scan_swizzle(&insn, rdep); if (is_scalar_op(insn.Instruction.Opcode)) { @@ -2109,7 +3106,7 @@ nv50_tgsi_insn(struct nv50_pc *pc, const union tgsi_full_token *tok) } pc->r_brdc = NULL; - if (!deqs) + if (!deqs || (!rdep[0] && !rdep[1] && !rdep[2] && !rdep[3])) return nv50_program_tx_insn(pc, &insn); deqs = nv50_revdep_reorder(m, rdep); @@ -2117,10 +3114,10 @@ nv50_tgsi_insn(struct nv50_pc *pc, const union tgsi_full_token *tok) for (i = 0; i < 4; ++i) { assert(pc->r_dst[m[i]] == NULL); - insn.FullDstRegisters[0].DstRegister.WriteMask = - fd->DstRegister.WriteMask & (1 << m[i]); + insn.Dst[0].Register.WriteMask = + fd->Register.WriteMask & (1 << m[i]); - if (!insn.FullDstRegisters[0].DstRegister.WriteMask) + if (!insn.Dst[0].Register.WriteMask) continue; if (deqs & (1 << i)) @@ -2160,7 +3157,7 @@ load_interpolant(struct nv50_pc *pc, struct nv50_reg *reg) iv->rhw = popcnt4(pc->p->cfg.regs[1] >> 24) - 1; emit_interp(pc, iv, NULL, mode & INTERP_CENTROID); - emit_flop(pc, 0, iv, iv); + emit_flop(pc, NV50_FLOP_RCP, iv, iv); /* XXX: when loading interpolants dynamically, move these * to the program head, or make sure it can't be skipped. @@ -2170,6 +3167,23 @@ load_interpolant(struct nv50_pc *pc, struct nv50_reg *reg) emit_interp(pc, reg, iv, mode); } +/* The face input is always at v[255] (varying space), with a + * value of 0 for back-facing, and 0xffffffff for front-facing. + */ +static void +load_frontfacing(struct nv50_pc *pc, struct nv50_reg *a) +{ + struct nv50_reg *one = alloc_immd(pc, 1.0f); + + assert(a->rhw == -1); + alloc_reg(pc, a); /* do this before rhw is set */ + a->rhw = 255; + load_interpolant(pc, a); + emit_bitop2(pc, a, a, one, TGSI_OPCODE_AND); + + FREE(one); +} + static boolean nv50_program_tx_prep(struct nv50_pc *pc) { @@ -2189,10 +3203,10 @@ nv50_program_tx_prep(struct nv50_pc *pc) const struct tgsi_full_immediate *imm = &tp.FullToken.FullImmediate; - ctor_immd(pc, imm->u[0].Float, - imm->u[1].Float, - imm->u[2].Float, - imm->u[3].Float); + ctor_immd_4f32(pc, imm->u[0].Float, + imm->u[1].Float, + imm->u[2].Float, + imm->u[3].Float); } break; case TGSI_TOKEN_TYPE_DECLARATION: @@ -2201,8 +3215,8 @@ nv50_program_tx_prep(struct nv50_pc *pc) unsigned si, last, first, mode; d = &tp.FullToken.FullDeclaration; - first = d->DeclarationRange.First; - last = d->DeclarationRange.Last; + first = d->Range.First; + last = d->Range.Last; switch (d->Declaration.File) { case TGSI_FILE_TEMPORARY: @@ -2212,8 +3226,8 @@ nv50_program_tx_prep(struct nv50_pc *pc) p->type == PIPE_SHADER_FRAGMENT) break; - si = d->Semantic.SemanticIndex; - switch (d->Semantic.SemanticName) { + si = d->Semantic.Index; + switch (d->Semantic.Name) { case TGSI_SEMANTIC_BCOLOR: p->cfg.two_side[si].hw = first; if (p->cfg.io_nr > first) @@ -2224,6 +3238,9 @@ nv50_program_tx_prep(struct nv50_pc *pc) if (p->cfg.io_nr > first) p->cfg.io_nr = first; break; + case TGSI_SEMANTIC_EDGEFLAG: + pc->edgeflag_out = first; + break; /* case TGSI_SEMANTIC_CLIP_DISTANCE: p->cfg.clpd = MIN2(p->cfg.clpd, first); @@ -2259,8 +3276,8 @@ nv50_program_tx_prep(struct nv50_pc *pc) pc->interp_mode[i] = mode; } break; + case TGSI_FILE_ADDRESS: case TGSI_FILE_CONSTANT: - break; case TGSI_FILE_SAMPLER: break; default: @@ -2291,7 +3308,7 @@ nv50_program_tx_prep(struct nv50_pc *pc) for (i = 0, rid = 0; i < pc->result_nr; ++i) { p->cfg.io[i].hw = rid; - p->cfg.io[i].id_vp = i; + p->cfg.io[i].id = i; for (c = 0; c < 4; ++c) { int n = i * 4 + c; @@ -2314,6 +3331,8 @@ nv50_program_tx_prep(struct nv50_pc *pc) int rid, aid; unsigned n = 0, m = pc->attr_nr - flat_nr; + pc->allow32 = TRUE; + int base = (TGSI_SEMANTIC_POSITION == p->info.input_semantic_name[0]) ? 0 : 1; @@ -2321,14 +3340,12 @@ nv50_program_tx_prep(struct nv50_pc *pc) * the lower hardware IDs, so sort them: */ for (i = 0; i < pc->attr_nr; i++) { - if (pc->interp_mode[i] == INTERP_FLAT) { - p->cfg.io[m].id_vp = i + base; - p->cfg.io[m++].id_fp = i; - } else { + if (pc->interp_mode[i] == INTERP_FLAT) + p->cfg.io[m++].id = i; + else { if (!(pc->interp_mode[i] & INTERP_PERSPECTIVE)) p->cfg.io[n].linear = TRUE; - p->cfg.io[n].id_vp = i + base; - p->cfg.io[n++].id_fp = i; + p->cfg.io[n++].id = i; } } @@ -2340,7 +3357,13 @@ nv50_program_tx_prep(struct nv50_pc *pc) for (n = 0; n < pc->attr_nr; ++n) { p->cfg.io[n].hw = rid = aid; - i = p->cfg.io[n].id_fp; + i = p->cfg.io[n].id; + + if (p->info.input_semantic_name[n] == + TGSI_SEMANTIC_FACE) { + load_frontfacing(pc, &pc->attr[i * 4]); + continue; + } for (c = 0; c < 4; ++c) { if (!pc->attr[i * 4 + c].acc) @@ -2374,8 +3397,8 @@ nv50_program_tx_prep(struct nv50_pc *pc) for (i = 0; i < pc->attr_nr; i++) { ubyte si, sn; - sn = p->info.input_semantic_name[p->cfg.io[i].id_fp]; - si = p->info.input_semantic_index[p->cfg.io[i].id_fp]; + sn = p->info.input_semantic_name[p->cfg.io[i].id]; + si = p->info.input_semantic_index[p->cfg.io[i].id]; if (sn == TGSI_SEMANTIC_COLOR) { p->cfg.two_side[si] = p->cfg.io[i]; @@ -2400,6 +3423,10 @@ nv50_program_tx_prep(struct nv50_pc *pc) pc->result[2].rhw = rid; p->cfg.high_result = rid; + + /* separate/different colour results for MRTs ? */ + if (pc->result_nr - (p->info.writes_z ? 1 : 0) > 1) + p->cfg.regs[2] |= 1; } if (pc->immd_nr) { @@ -2454,12 +3481,16 @@ ctor_nv50_pc(struct nv50_pc *pc, struct nv50_program *p) pc->attr_nr = p->info.file_max[TGSI_FILE_INPUT] + 1; pc->result_nr = p->info.file_max[TGSI_FILE_OUTPUT] + 1; pc->param_nr = p->info.file_max[TGSI_FILE_CONSTANT] + 1; + pc->addr_nr = p->info.file_max[TGSI_FILE_ADDRESS] + 1; + assert(pc->addr_nr <= 2); p->cfg.high_temp = 4; p->cfg.two_side[0].hw = 0x40; p->cfg.two_side[1].hw = 0x40; + p->cfg.edgeflag_in = pc->edgeflag_out = 0xff; + switch (p->type) { case PIPE_SHADER_VERTEX: p->cfg.psiz = 0x40; @@ -2522,31 +3553,21 @@ ctor_nv50_pc(struct nv50_pc *pc, struct nv50_program *p) ctor_reg(&pc->param[rid], P_CONST, i, rid); } - return TRUE; -} - -static void -nv50_fp_move_results(struct nv50_pc *pc) -{ - struct nv50_reg reg; - unsigned i; - - ctor_reg(®, P_TEMP, -1, -1); - - for (i = 0; i < pc->result_nr * 4; ++i) { - if (pc->result[i].rhw < 0 || pc->result[i].hw < 0) - continue; - if (pc->result[i].rhw != pc->result[i].hw) { - reg.hw = pc->result[i].rhw; - emit_mov(pc, ®, &pc->result[i]); - } + if (pc->addr_nr) { + pc->addr = CALLOC(pc->addr_nr * 4, sizeof(struct nv50_reg *)); + if (!pc->addr) + return FALSE; } + for (i = 0; i < NV50_SU_MAX_ADDR; ++i) + ctor_reg(&pc->r_addr[i], P_ADDR, -256, i + 1); + + return TRUE; } static void nv50_program_fixup_insns(struct nv50_pc *pc) { - struct nv50_program_exec *e, *prev = NULL, **bra_list; + struct nv50_program_exec *e, **bra_list; unsigned i, n, pos; bra_list = CALLOC(pc->p->exec_size, sizeof(struct nv50_program_exec *)); @@ -2566,27 +3587,24 @@ nv50_program_fixup_insns(struct nv50_pc *pc) for (i = 0; i < n; ++i) if (bra_list[i]->param.index >= pos) bra_list[i]->param.index += 1; + for (i = 0; i < pc->insn_nr; ++i) + if (pc->insn_pos[i] >= pos) + pc->insn_pos[i] += 1; convert_to_long(pc, e); ++pos; } - if (e->next) - prev = e; } - assert(!is_immd(pc->p->exec_head)); - assert(!is_immd(pc->p->exec_tail)); + FREE(bra_list); - /* last instruction must be long so it can have the end bit set */ - if (!is_long(pc->p->exec_tail)) { - convert_to_long(pc, pc->p->exec_tail); - if (prev) - convert_to_long(pc, prev); - } - assert(!(pc->p->exec_tail->inst[1] & 2)); - /* set the end-bit */ - pc->p->exec_tail->inst[1] |= 1; + if (!pc->p->info.opcode_count[TGSI_OPCODE_CAL]) + return; - FREE(bra_list); + /* fill in CALL offsets */ + for (e = pc->p->exec_head; e; e = e->next) { + if ((e->inst[0] & 2) && (e->inst[0] >> 28) == 0x2) + e->param.index = pc->insn_pos[e->param.index]; + } } static boolean @@ -2608,19 +3626,20 @@ nv50_program_tx(struct nv50_program *p) if (ret == FALSE) goto out_cleanup; + pc->insn_pos = MALLOC(pc->insn_nr * sizeof(unsigned)); + tgsi_parse_init(&parse, pc->p->pipe.tokens); while (!tgsi_parse_end_of_tokens(&parse)) { const union tgsi_full_token *tok = &parse.FullToken; - /* don't allow half insn/immd on first and last instruction */ + /* previously allow32 was FALSE for first & last instruction */ pc->allow32 = TRUE; - if (pc->insn_cur == 0 || pc->insn_cur + 2 == pc->insn_nr) - pc->allow32 = FALSE; tgsi_parse_token(&parse); switch (tok->Token.Type) { case TGSI_TOKEN_TYPE_INSTRUCTION: + pc->insn_pos[pc->insn_cur] = pc->p->exec_size; ++pc->insn_cur; ret = nv50_tgsi_insn(pc, tok); if (ret == FALSE) @@ -2631,9 +3650,6 @@ nv50_program_tx(struct nv50_program *p) } } - if (pc->p->type == PIPE_SHADER_FRAGMENT) - nv50_fp_move_results(pc); - nv50_program_fixup_insns(pc); p->param_nr = pc->param_nr * 4; @@ -2657,7 +3673,7 @@ nv50_program_validate(struct nv50_context *nv50, struct nv50_program *p) } static void -nv50_program_upload_data(struct nv50_context *nv50, float *map, +nv50_program_upload_data(struct nv50_context *nv50, uint32_t *map, unsigned start, unsigned count, unsigned cbuf) { struct nouveau_channel *chan = nv50->screen->base.channel; @@ -2701,12 +3717,12 @@ nv50_program_validate_data(struct nv50_context *nv50, struct nv50_program *p) p->immd_nr, NV50_CB_PMISC); } - assert(p->param_nr <= 128); + assert(p->param_nr <= 512); if (p->param_nr) { unsigned cb; - float *map = pipe_buffer_map(pscreen, nv50->constbuf[p->type], - PIPE_BUFFER_USAGE_CPU_READ); + uint32_t *map = pipe_buffer_map(pscreen, nv50->constbuf[p->type], + PIPE_BUFFER_USAGE_CPU_READ); if (p->type == PIPE_SHADER_VERTEX) cb = NV50_CB_PVP; @@ -2722,11 +3738,8 @@ static void nv50_program_validate_code(struct nv50_context *nv50, struct nv50_program *p) { struct nouveau_channel *chan = nv50->screen->base.channel; - struct nouveau_grobj *tesla = nv50->screen->tesla; struct nv50_program_exec *e; - struct nouveau_stateobj *so; - const unsigned flags = NOUVEAU_BO_VRAM | NOUVEAU_BO_WR; - unsigned start, count, *up, *ptr; + uint32_t *up, i; boolean upload = FALSE; if (!p->bo) { @@ -2741,32 +3754,37 @@ nv50_program_validate_code(struct nv50_context *nv50, struct nv50_program *p) if (!upload) return; - for (e = p->exec_head; e; e = e->next) { + up = MALLOC(p->exec_size * 4); + + for (i = 0, e = p->exec_head; e; e = e->next) { unsigned ei, ci, bs; - if (e->param.index < 0) - continue; + if (e->param.index >= 0 && e->param.mask) { + bs = (e->inst[1] >> 22) & 0x07; + assert(bs < 2); + ei = e->param.shift >> 5; + ci = e->param.index; + if (bs == 0) + ci += p->data[bs]->start; - if (e->param.mask == 0) { + e->inst[ei] &= ~e->param.mask; + e->inst[ei] |= (ci << e->param.shift); + } else + if (e->param.index >= 0) { + /* zero mask means param is a jump/branch offset */ assert(!(e->param.index & 1)); /* seem to be 8 byte steps */ ei = (e->param.index >> 1) + 0 /* START_ID */; e->inst[0] &= 0xf0000fff; e->inst[0] |= ei << 12; - continue; } - bs = (e->inst[1] >> 22) & 0x07; - assert(bs < 2); - ei = e->param.shift >> 5; - ci = e->param.index; - if (bs == 0) - ci += p->data[bs]->start; - - e->inst[ei] &= ~e->param.mask; - e->inst[ei] |= (ci << e->param.shift); + up[i++] = e->inst[0]; + if (is_long(e)) + up[i++] = e->inst[1]; } + assert(i == p->exec_size); if (p->data[0]) p->data_start[0] = p->data[0]->start; @@ -2779,45 +3797,12 @@ nv50_program_validate_code(struct nv50_context *nv50, struct nv50_program *p) NOUVEAU_ERR("0x%08x\n", e->inst[1]); } #endif - - up = ptr = MALLOC(p->exec_size * 4); - for (e = p->exec_head; e; e = e->next) { - *(ptr++) = e->inst[0]; - if (is_long(e)) - *(ptr++) = e->inst[1]; - } - - so = so_new(4,2); - so_method(so, nv50->screen->tesla, NV50TCL_CB_DEF_ADDRESS_HIGH, 3); - so_reloc (so, p->bo, 0, flags | NOUVEAU_BO_HIGH, 0, 0); - so_reloc (so, p->bo, 0, flags | NOUVEAU_BO_LOW, 0, 0); - so_data (so, (NV50_CB_PUPLOAD << 16) | 0x0800); //(p->exec_size * 4)); - - start = 0; count = p->exec_size; - while (count) { - struct nouveau_channel *chan = nv50->screen->base.channel; - unsigned nr; - - so_emit(chan, so); - - nr = MIN2(count, 2047); - nr = MIN2(chan->pushbuf->remaining, nr); - if (chan->pushbuf->remaining < (nr + 3)) { - FIRE_RING(chan); - continue; - } - - BEGIN_RING(chan, tesla, NV50TCL_CB_ADDR, 1); - OUT_RING (chan, (start << 8) | NV50_CB_PUPLOAD); - BEGIN_RING(chan, tesla, NV50TCL_CB_DATA(0) | 0x40000000, nr); - OUT_RINGp (chan, up + start, nr); - - start += nr; - count -= nr; - } + nv50_upload_sifc(nv50, p->bo, 0, NOUVEAU_BO_VRAM, + NV50_2D_DST_FORMAT_R8_UNORM, 65536, 1, 262144, + up, NV50_2D_SIFC_FORMAT_R8_UNORM, 0, + 0, 0, p->exec_size * 4, 1, 1); FREE(up); - so_ref(NULL, &so); } void @@ -2836,7 +3821,7 @@ nv50_vertprog_validate(struct nv50_context *nv50) nv50_program_validate_data(nv50, p); nv50_program_validate_code(nv50, p); - so = so_new(13, 2); + so = so_new(5, 8, 2); so_method(so, tesla, NV50TCL_VP_ADDRESS_HIGH, 2); so_reloc (so, p->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD | NOUVEAU_BO_HIGH, 0, 0); @@ -2872,7 +3857,7 @@ nv50_fragprog_validate(struct nv50_context *nv50) nv50_program_validate_data(nv50, p); nv50_program_validate_code(nv50, p); - so = so_new(64, 2); + so = so_new(6, 7, 2); so_method(so, tesla, NV50TCL_FP_ADDRESS_HIGH, 2); so_reloc (so, p->bo, 0, NOUVEAU_BO_VRAM | NOUVEAU_BO_RD | NOUVEAU_BO_HIGH, 0, 0); @@ -2882,7 +3867,7 @@ nv50_fragprog_validate(struct nv50_context *nv50) so_data (so, p->cfg.high_temp); so_method(so, tesla, NV50TCL_FP_RESULT_COUNT, 1); so_data (so, p->cfg.high_result); - so_method(so, tesla, NV50TCL_FP_CTRL_UNK19A8, 1); + so_method(so, tesla, NV50TCL_FP_CONTROL, 1); so_data (so, p->cfg.regs[2]); so_method(so, tesla, NV50TCL_FP_CTRL_UNK196C, 1); so_data (so, p->cfg.regs[3]); @@ -2899,15 +3884,15 @@ nv50_pntc_replace(struct nv50_context *nv50, uint32_t pntc[8], unsigned base) struct nv50_program *vp = nv50->vertprog; unsigned i, c, m = base; - /* XXX: This can't work correctly in all cases yet, we either - * have to create TGSI_SEMANTIC_PNTC or sprite_coord_mode has - * to be per FP input instead of per VP output + /* XXX: this might not work correctly in all cases yet - we'll + * just assume that an FP generic input that is not written in + * the VP is PointCoord. */ memset(pntc, 0, 8 * sizeof(uint32_t)); for (i = 0; i < fp->cfg.io_nr; i++) { uint8_t sn, si; - uint8_t j = fp->cfg.io[i].id_vp, k = fp->cfg.io[i].id_fp; + uint8_t j, k = fp->cfg.io[i].id; unsigned n = popcnt4(fp->cfg.io[i].mask); if (fp->info.input_semantic_name[k] != TGSI_SEMANTIC_GENERIC) { @@ -2915,10 +3900,16 @@ nv50_pntc_replace(struct nv50_context *nv50, uint32_t pntc[8], unsigned base) continue; } - sn = vp->info.input_semantic_name[j]; - si = vp->info.input_semantic_index[j]; + for (j = 0; j < vp->info.num_outputs; ++j) { + sn = vp->info.output_semantic_name[j]; + si = vp->info.output_semantic_index[j]; - if (j < fp->cfg.io_nr && sn == TGSI_SEMANTIC_GENERIC) { + if (sn == fp->info.input_semantic_name[k] && + si == fp->info.input_semantic_index[k]) + break; + } + + if (j < vp->info.num_outputs) { ubyte mode = nv50->rasterizer->pipe.sprite_coord_mode[si]; @@ -3006,20 +3997,24 @@ nv50_linkage_validate(struct nv50_context *nv50) reg[0] += m - 4; /* adjust FFC0 id */ reg[4] |= m << 8; /* set mid where 'normal' FP inputs start */ - i = 0; - if (fp->info.input_semantic_name[0] == TGSI_SEMANTIC_POSITION) - i = 1; - for (; i < fp->cfg.io_nr; i++) { - ubyte sn = fp->info.input_semantic_name[fp->cfg.io[i].id_fp]; - ubyte si = fp->info.input_semantic_index[fp->cfg.io[i].id_fp]; - - n = fp->cfg.io[i].id_vp; - if (n >= vp->cfg.io_nr || - vp->info.output_semantic_name[n] != sn || - vp->info.output_semantic_index[n] != si) - vpo = &dummy; - else - vpo = &vp->cfg.io[n]; + for (i = 0; i < fp->cfg.io_nr; i++) { + ubyte sn = fp->info.input_semantic_name[fp->cfg.io[i].id]; + ubyte si = fp->info.input_semantic_index[fp->cfg.io[i].id]; + + /* position must be mapped first */ + assert(i == 0 || sn != TGSI_SEMANTIC_POSITION); + + /* maybe even remove these from cfg.io */ + if (sn == TGSI_SEMANTIC_POSITION || sn == TGSI_SEMANTIC_FACE) + continue; + + /* VP outputs and vp->cfg.io are in the same order */ + for (n = 0; n < vp->info.num_outputs; ++n) { + if (vp->info.output_semantic_name[n] == sn && + vp->info.output_semantic_index[n] == si) + break; + } + vpo = (n < vp->info.num_outputs) ? &vp->cfg.io[n] : &dummy; m = nv50_sreg4_map(map, m, lin, &fp->cfg.io[i], vpo); } @@ -3030,7 +4025,7 @@ nv50_linkage_validate(struct nv50_context *nv50) } /* now fill the stateobj */ - so = so_new(64, 0); + so = so_new(6, 58, 0); n = (m + 3) / 4; so_method(so, tesla, NV50TCL_VP_RESULT_MAP_SIZE, 1); @@ -3044,7 +4039,7 @@ nv50_linkage_validate(struct nv50_context *nv50) so_method(so, tesla, NV50TCL_FP_INTERPOLANT_CTRL, 1); so_data (so, reg[4]); - so_method(so, tesla, 0x1540, 4); + so_method(so, tesla, NV50TCL_NOPERSPECTIVE_BITMAP(0), 4); so_datap (so, lin, 4); if (nv50->rasterizer->pipe.point_sprite) { |