Merge commit 'origin/master' into gallium-msaa

11 files changed, 209 insertions, 54 deletions

diff --git a/src/gallium/auxiliary/draw/draw_llvm.c b/src/gallium/auxiliary/draw/draw_llvm.c
index 703b46b518..3b2df054c3 100644
--- a/src/gallium/auxiliary/draw/draw_llvm.c
+++ b/src/gallium/auxiliary/draw/draw_llvm.c
@@ -24,6 +24,8 @@
 /* generates the draw jit function */
 static void
 draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *var);
+static void
+draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *var);
 
 static void
 init_globals(struct draw_llvm *llvm)
@@ -218,6 +220,7 @@ draw_llvm_prepare(struct draw_llvm *llvm, int num_inputs)
    llvm->vertex_header_ptr_type = create_vertex_header(llvm, num_inputs);
 
    draw_llvm_generate(llvm, variant);
+   draw_llvm_generate_elts(llvm, variant);
 
    return variant;
 }
@@ -696,6 +699,158 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
       lp_disassemble(variant->jit_func);
 }
 
+
+static void
+draw_llvm_generate_elts(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
+{
+   LLVMTypeRef arg_types[7];
+   LLVMTypeRef func_type;
+   LLVMValueRef context_ptr;
+   LLVMBasicBlockRef block;
+   LLVMBuilderRef builder;
+   LLVMValueRef fetch_elts, fetch_count, stride, step, io_itr;
+   LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
+   struct draw_context *draw = llvm->draw;
+   unsigned i, j;
+   struct lp_build_context bld;
+   struct lp_build_context bld_int;
+   struct lp_build_loop_state lp_loop;
+   struct lp_type vs_type = lp_type_float_vec(32);
+   const int max_vertices = 4;
+   LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][NUM_CHANNELS];
+   LLVMValueRef fetch_max;
+
+   arg_types[0] = llvm->context_ptr_type;               /* context */
+   arg_types[1] = llvm->vertex_header_ptr_type;         /* vertex_header */
+   arg_types[2] = llvm->buffer_ptr_type;                /* vbuffers */
+   arg_types[3] = LLVMPointerType(LLVMInt32Type(), 0);  /* fetch_elts * */
+   arg_types[4] = LLVMInt32Type();                      /* fetch_count */
+   arg_types[5] = LLVMInt32Type();                      /* stride */
+   arg_types[6] = llvm->vb_ptr_type;                    /* pipe_vertex_buffer's */
+
+   func_type = LLVMFunctionType(LLVMVoidType(), arg_types, Elements(arg_types), 0);
+
+   variant->function_elts = LLVMAddFunction(llvm->module, "draw_llvm_shader_elts", func_type);
+   LLVMSetFunctionCallConv(variant->function_elts, LLVMCCallConv);
+   for(i = 0; i < Elements(arg_types); ++i)
+      if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
+         LLVMAddAttribute(LLVMGetParam(variant->function_elts, i), LLVMNoAliasAttribute);
+
+   context_ptr  = LLVMGetParam(variant->function_elts, 0);
+   io_ptr       = LLVMGetParam(variant->function_elts, 1);
+   vbuffers_ptr = LLVMGetParam(variant->function_elts, 2);
+   fetch_elts   = LLVMGetParam(variant->function_elts, 3);
+   fetch_count  = LLVMGetParam(variant->function_elts, 4);
+   stride       = LLVMGetParam(variant->function_elts, 5);
+   vb_ptr       = LLVMGetParam(variant->function_elts, 6);
+
+   lp_build_name(context_ptr, "context");
+   lp_build_name(io_ptr, "io");
+   lp_build_name(vbuffers_ptr, "vbuffers");
+   lp_build_name(fetch_elts, "fetch_elts");
+   lp_build_name(fetch_count, "fetch_count");
+   lp_build_name(stride, "stride");
+   lp_build_name(vb_ptr, "vb");
+
+   /*
+    * Function body
+    */
+
+   block = LLVMAppendBasicBlock(variant->function_elts, "entry");
+   builder = LLVMCreateBuilder();
+   LLVMPositionBuilderAtEnd(builder, block);
+
+   lp_build_context_init(&bld, builder, vs_type);
+   lp_build_context_init(&bld_int, builder, lp_type_int(32));
+
+   step = LLVMConstInt(LLVMInt32Type(), max_vertices, 0);
+
+   fetch_max = LLVMBuildSub(builder, fetch_count,
+                            LLVMConstInt(LLVMInt32Type(), 1, 0),
+                            "fetch_max");
+
+   lp_build_loop_begin(builder, LLVMConstInt(LLVMInt32Type(), 0, 0), &lp_loop);
+   {
+      LLVMValueRef inputs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS];
+      LLVMValueRef aos_attribs[PIPE_MAX_SHADER_INPUTS][NUM_CHANNELS] = { { 0 } };
+      LLVMValueRef io;
+      const LLVMValueRef (*ptr_aos)[NUM_CHANNELS];
+
+      io_itr = lp_loop.counter;
+      io = LLVMBuildGEP(builder, io_ptr, &io_itr, 1, "");
+#if DEBUG_STORE
+      lp_build_printf(builder, " --- io %d = %p, loop counter %d\n",
+                      io_itr, io, lp_loop.counter);
+#endif
+      for (i = 0; i < NUM_CHANNELS; ++i) {
+         LLVMValueRef true_index = LLVMBuildAdd(
+            builder,
+            lp_loop.counter,
+            LLVMConstInt(LLVMInt32Type(), i, 0), "");
+         LLVMValueRef fetch_ptr;
+
+         /* make sure we're not out of bounds which can happen
+          * if fetch_count % 4 != 0, because on the last iteration
+          * a few of the 4 vertex fetches will be out of bounds */
+         true_index = lp_build_min(&bld_int, true_index, fetch_max);
+
+         fetch_ptr = LLVMBuildGEP(builder, fetch_elts,
+                                  &true_index, 1, "");
+         true_index = LLVMBuildLoad(builder, fetch_ptr, "fetch_elt");
+         for (j = 0; j < draw->pt.nr_vertex_elements; ++j) {
+            struct pipe_vertex_element *velem = &draw->pt.vertex_element[j];
+            LLVMValueRef vb_index = LLVMConstInt(LLVMInt32Type(),
+                                                 velem->vertex_buffer_index,
+                                                 0);
+            LLVMValueRef vb = LLVMBuildGEP(builder, vb_ptr,
+                                           &vb_index, 1, "");
+            generate_fetch(builder, vbuffers_ptr,
+                           &aos_attribs[j][i], velem, vb, true_index);
+         }
+      }
+      convert_to_soa(builder, aos_attribs, inputs,
+                     draw->pt.nr_vertex_elements);
+
+      ptr_aos = (const LLVMValueRef (*)[NUM_CHANNELS]) inputs;
+      generate_vs(llvm,
+                  builder,
+                  outputs,
+                  ptr_aos,
+                  context_ptr);
+
+      convert_to_aos(builder, io, outputs,
+                     draw->vs.vertex_shader->info.num_outputs,
+                     max_vertices);
+   }
+   lp_build_loop_end_cond(builder, fetch_count, step, LLVMIntUGE, &lp_loop);
+
+   LLVMBuildRetVoid(builder);
+
+   LLVMDisposeBuilder(builder);
+
+   /*
+    * Translate the LLVM IR into machine code.
+    */
+#ifdef DEBUG
+   if(LLVMVerifyFunction(variant->function_elts, LLVMPrintMessageAction)) {
+      LLVMDumpValue(variant->function_elts);
+      assert(0);
+   }
+#endif
+
+   LLVMRunFunctionPassManager(llvm->pass, variant->function_elts);
+
+   if (0) {
+      LLVMDumpValue(variant->function_elts);
+      debug_printf("\n");
+   }
+   variant->jit_func_elts = (draw_jit_vert_func_elts)LLVMGetPointerToGlobal(
+      llvm->draw->engine, variant->function_elts);
+
+   if (0)
+      lp_disassemble(variant->jit_func_elts);
+}
+
 void
 draw_llvm_make_variant_key(struct draw_llvm *llvm,
                            struct draw_llvm_variant_key *key)
diff --git a/src/gallium/auxiliary/draw/draw_llvm.h b/src/gallium/auxiliary/draw/draw_llvm.h
index 20f4f3da5f..58fee7f9d6 100644
--- a/src/gallium/auxiliary/draw/draw_llvm.h
+++ b/src/gallium/auxiliary/draw/draw_llvm.h
@@ -94,6 +94,16 @@ typedef void
                       unsigned stride,
                       struct pipe_vertex_buffer *vertex_buffers);
 
+
+typedef void
+(*draw_jit_vert_func_elts)(struct draw_jit_context *context,
+                           struct vertex_header *io,
+                           const char *vbuffers[PIPE_MAX_ATTRIBS],
+                           const unsigned *fetch_elts,
+                           unsigned fetch_count,
+                           unsigned stride,
+                           struct pipe_vertex_buffer *vertex_buffers);
+
 struct draw_llvm {
    struct draw_context *draw;
 
@@ -122,7 +132,9 @@ struct draw_llvm_variant
 {
    struct draw_llvm_variant_key key;
    LLVMValueRef function;
+   LLVMValueRef function_elts;
    draw_jit_vert_func jit_func;
+   draw_jit_vert_func_elts jit_func_elts;
 
    struct draw_llvm_variant *next;
 };
diff --git a/src/gallium/auxiliary/draw/draw_pt.c b/src/gallium/auxiliary/draw/draw_pt.c
index 1d9395e06c..b853f3a89f 100644
--- a/src/gallium/auxiliary/draw/draw_pt.c
+++ b/src/gallium/auxiliary/draw/draw_pt.c
@@ -38,10 +38,10 @@
 #include "util/u_prim.h"
 
 
-DEBUG_GET_ONCE_BOOL_OPTION(draw_fse, "DRAW_FSE", FALSE);
-DEBUG_GET_ONCE_BOOL_OPTION(draw_no_fse, "DRAW_NO_FSE", FALSE);
+DEBUG_GET_ONCE_BOOL_OPTION(draw_fse, "DRAW_FSE", FALSE)
+DEBUG_GET_ONCE_BOOL_OPTION(draw_no_fse, "DRAW_NO_FSE", FALSE)
 #ifdef HAVE_LLVM
-DEBUG_GET_ONCE_BOOL_OPTION(draw_use_llvm, "DRAW_USE_LLVM", TRUE);
+DEBUG_GET_ONCE_BOOL_OPTION(draw_use_llvm, "DRAW_USE_LLVM", TRUE)
 #endif
 
 static unsigned trim( unsigned count, unsigned first, unsigned incr )
diff --git a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
index f71271bd91..d2a492f2b4 100644
--- a/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
+++ b/src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c
@@ -167,8 +167,6 @@ static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
 {
    struct llvm_middle_end *fpme = (struct llvm_middle_end *)middle;
    struct draw_context *draw = fpme->draw;
-   struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
-   struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
    unsigned opt = fpme->opt;
    unsigned alloc_count = align( fetch_count, 4 );
 
@@ -182,35 +180,13 @@ static void llvm_middle_end_run( struct draw_pt_middle_end *middle,
       return;
    }
 
-   /* Fetch into our vertex buffer
-    */
-   draw_pt_fetch_run( fpme->fetch,
-		      fetch_elts,
-		      fetch_count,
-		      (char *)pipeline_verts );
-
-   /* Run the shader, note that this overwrites the data[] parts of
-    * the pipeline verts.  If there is no shader, eg if
-    * bypass_vs_clip_and_viewport, then the inputs == outputs, and are
-    * already in the correct place.*/
-   if (opt & PT_SHADE)
-   {
-      vshader->run_linear(vshader,
-                          (const float (*)[4])pipeline_verts->data,
-                          (      float (*)[4])pipeline_verts->data,
-                          draw->pt.user.vs_constants,
-                          fetch_count,
-                          fpme->vertex_size,
-                          fpme->vertex_size);
-      if (gshader)
-         draw_geometry_shader_run(gshader,
-                                  (const float (*)[4])pipeline_verts->data,
-                                  (      float (*)[4])pipeline_verts->data,
-                                  draw->pt.user.gs_constants,
-                                  fetch_count,
-                                  fpme->vertex_size,
-                                  fpme->vertex_size);
-   }
+   fpme->current_variant->jit_func_elts( &fpme->llvm->jit_context,
+                                         pipeline_verts,
+                                         (const char **)draw->pt.user.vbuffer,
+                                         fetch_elts,
+                                         fetch_count,
+                                         fpme->vertex_size,
+                                         draw->pt.vertex_buffer );
 
    if (draw_pt_post_vs_run( fpme->post_vs,
 			    pipeline_verts,
diff --git a/src/gallium/auxiliary/draw/draw_vs.c b/src/gallium/auxiliary/draw/draw_vs.c
index 0c05957350..c2832eefa2 100644
--- a/src/gallium/auxiliary/draw/draw_vs.c
+++ b/src/gallium/auxiliary/draw/draw_vs.c
@@ -46,7 +46,7 @@
 #include "tgsi/tgsi_dump.h"
 #include "tgsi/tgsi_exec.h"
 
-DEBUG_GET_ONCE_BOOL_OPTION(gallium_dump_vs, "GALLIUM_DUMP_VS", FALSE);
+DEBUG_GET_ONCE_BOOL_OPTION(gallium_dump_vs, "GALLIUM_DUMP_VS", FALSE)
 
 void
 draw_vs_set_constants(struct draw_context *draw,
diff --git a/src/gallium/auxiliary/draw/draw_vs.h b/src/gallium/auxiliary/draw/draw_vs.h
index efead42335..6c7e94db43 100644
--- a/src/gallium/auxiliary/draw/draw_vs.h
+++ b/src/gallium/auxiliary/draw/draw_vs.h
@@ -169,8 +169,9 @@ draw_create_vs_ppc(struct draw_context *draw,
 struct draw_vs_varient_key;
 struct draw_vertex_shader;
 
-struct draw_vs_varient *draw_vs_varient_aos_sse( struct draw_vertex_shader *vs,
-                                                 const struct draw_vs_varient_key *key );
+struct draw_vs_varient *
+draw_vs_create_varient_aos_sse( struct draw_vertex_shader *vs,
+                                const struct draw_vs_varient_key *key );
 
 
 
@@ -188,8 +189,9 @@ struct translate *draw_vs_get_fetch( struct draw_context *draw,
 struct translate *draw_vs_get_emit( struct draw_context *draw,
                                     struct translate_key *key );
 
-struct draw_vs_varient *draw_vs_varient_generic( struct draw_vertex_shader *vs,
-                                                 const struct draw_vs_varient_key *key );
+struct draw_vs_varient *
+draw_vs_create_varient_generic( struct draw_vertex_shader *vs,
+                                const struct draw_vs_varient_key *key );
 
 
 
diff --git a/src/gallium/auxiliary/draw/draw_vs_aos.c b/src/gallium/auxiliary/draw/draw_vs_aos.c
index e7121f3654..19f49e34c8 100644
--- a/src/gallium/auxiliary/draw/draw_vs_aos.c
+++ b/src/gallium/auxiliary/draw/draw_vs_aos.c
@@ -2089,13 +2089,21 @@ static boolean build_vertex_program( struct draw_vs_varient_aos_sse *varient,
 }
 
 
+/** cast wrapper */
+static INLINE struct draw_vs_varient_aos_sse *
+draw_vs_varient_aos_sse(struct draw_vs_varient *varient)
+{
+   return (struct draw_vs_varient_aos_sse *) varient;
+}
+
 
 static void vaos_set_buffer( struct draw_vs_varient *varient,
                              unsigned buf,
                              const void *ptr,
-                             unsigned stride )
+                             unsigned stride,
+                             unsigned max_stride)
 {
-   struct draw_vs_varient_aos_sse *vaos = (struct draw_vs_varient_aos_sse *)varient;
+   struct draw_vs_varient_aos_sse *vaos = draw_vs_varient_aos_sse(varient);
 
    if (buf < vaos->nr_vb) {
       vaos->buffer[buf].base_ptr = (char *)ptr;
@@ -2112,7 +2120,7 @@ static void PIPE_CDECL vaos_run_elts( struct draw_vs_varient *varient,
                                       unsigned count,
                                       void *output_buffer )
 {
-   struct draw_vs_varient_aos_sse *vaos = (struct draw_vs_varient_aos_sse *)varient;
+   struct draw_vs_varient_aos_sse *vaos = draw_vs_varient_aos_sse(varient);
    struct aos_machine *machine = vaos->draw->vs.aos_machine;
    unsigned i;
 
@@ -2136,7 +2144,7 @@ static void PIPE_CDECL vaos_run_linear( struct draw_vs_varient *varient,
                                         unsigned count,
                                         void *output_buffer )
 {
-   struct draw_vs_varient_aos_sse *vaos = (struct draw_vs_varient_aos_sse *)varient;
+   struct draw_vs_varient_aos_sse *vaos = draw_vs_varient_aos_sse(varient);
    struct aos_machine *machine = vaos->draw->vs.aos_machine;
    unsigned i;
 
@@ -2165,7 +2173,7 @@ static void PIPE_CDECL vaos_run_linear( struct draw_vs_varient *varient,
 
 static void vaos_destroy( struct draw_vs_varient *varient )
 {
-   struct draw_vs_varient_aos_sse *vaos = (struct draw_vs_varient_aos_sse *)varient;
+   struct draw_vs_varient_aos_sse *vaos = draw_vs_varient_aos_sse(varient);
 
    FREE( vaos->buffer );
 
@@ -2241,13 +2249,14 @@ static struct draw_vs_varient *varient_aos_sse( struct draw_vertex_shader *vs,
 }
 
 
-struct draw_vs_varient *draw_vs_varient_aos_sse( struct draw_vertex_shader *vs,
-                                                 const struct draw_vs_varient_key *key )
+struct draw_vs_varient *
+draw_vs_create_varient_aos_sse( struct draw_vertex_shader *vs,
+                                const struct draw_vs_varient_key *key )
 {
    struct draw_vs_varient *varient = varient_aos_sse( vs, key );
 
    if (varient == NULL) {
-      varient = draw_vs_varient_generic( vs, key );
+      varient = draw_vs_create_varient_generic( vs, key );
    }
 
    return varient;
diff --git a/src/gallium/auxiliary/draw/draw_vs_exec.c b/src/gallium/auxiliary/draw/draw_vs_exec.c
index 7deca2b69d..bc34d390da 100644
--- a/src/gallium/auxiliary/draw/draw_vs_exec.c
+++ b/src/gallium/auxiliary/draw/draw_vs_exec.c
@@ -203,7 +203,7 @@ draw_create_vs_exec(struct draw_context *draw,
    vs->base.prepare = vs_exec_prepare;
    vs->base.run_linear = vs_exec_run_linear;
    vs->base.delete = vs_exec_delete;
-   vs->base.create_varient = draw_vs_varient_generic;
+   vs->base.create_varient = draw_vs_create_varient_generic;
    vs->machine = draw->vs.machine;
 
    return &vs->base;
diff --git a/src/gallium/auxiliary/draw/draw_vs_ppc.c b/src/gallium/auxiliary/draw/draw_vs_ppc.c
index d869eecec5..5df84916c5 100644
--- a/src/gallium/auxiliary/draw/draw_vs_ppc.c
+++ b/src/gallium/auxiliary/draw/draw_vs_ppc.c
@@ -125,7 +125,7 @@ vs_ppc_run_linear( struct draw_vertex_shader *base,
        */
       shader->func(inputs_soa, outputs_soa, temps_soa,
 		   (float (*)[4]) shader->base.immediates,
-                   (const float (*)[4])constants[0],
+                   (float (*)[4])constants[0],
                    ppc_builtin_constants);
 
       /* convert (up to) four output verts from SoA back to AoS format */
@@ -190,7 +190,7 @@ draw_create_vs_ppc(struct draw_context *draw,
       vs->base.create_varient = draw_vs_varient_aos_ppc;
    else
 #endif
-      vs->base.create_varient = draw_vs_varient_generic;
+      vs->base.create_varient = draw_vs_create_varient_generic;
    vs->base.prepare = vs_ppc_prepare;
    vs->base.run_linear = vs_ppc_run_linear;
    vs->base.delete = vs_ppc_delete;
diff --git a/src/gallium/auxiliary/draw/draw_vs_sse.c b/src/gallium/auxiliary/draw/draw_vs_sse.c
index 54e6423388..14c95082a9 100644
--- a/src/gallium/auxiliary/draw/draw_vs_sse.c
+++ b/src/gallium/auxiliary/draw/draw_vs_sse.c
@@ -165,9 +165,9 @@ draw_create_vs_sse(struct draw_context *draw,
 
    vs->base.draw = draw;
    if (1)
-      vs->base.create_varient = draw_vs_varient_aos_sse;
+      vs->base.create_varient = draw_vs_create_varient_aos_sse;
    else
-      vs->base.create_varient = draw_vs_varient_generic;
+      vs->base.create_varient = draw_vs_create_varient_generic;
    vs->base.prepare = vs_sse_prepare;
    vs->base.run_linear = vs_sse_run_linear;
    vs->base.delete = vs_sse_delete;
diff --git a/src/gallium/auxiliary/draw/draw_vs_varient.c b/src/gallium/auxiliary/draw/draw_vs_varient.c
index 0abd827b61..6eb26927f2 100644
--- a/src/gallium/auxiliary/draw/draw_vs_varient.c
+++ b/src/gallium/auxiliary/draw/draw_vs_varient.c
@@ -263,8 +263,9 @@ static void vsvg_destroy( struct draw_vs_varient *varient )
 }
 
 
-struct draw_vs_varient *draw_vs_varient_generic( struct draw_vertex_shader *vs,
-                                                 const struct draw_vs_varient_key *key )
+struct draw_vs_varient *
+draw_vs_create_varient_generic( struct draw_vertex_shader *vs,
+                                const struct draw_vs_varient_key *key )
 {
    unsigned i;
    struct translate_key fetch, emit;