gallivm/llvmpipe: replace 'int stride' with 'int row_stride[MAX_LEVELS]'

The stride depends on the mipmap level.  Rename to row_stride to
distinguish from img_stride for 3D textures.

Fixes incorrect texel addressing in small mipmap levels.

6 files changed, 54 insertions, 20 deletions

diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample.h b/src/gallium/auxiliary/gallivm/lp_bld_sample.h
index 9e88ea54d7..7f08bfaac1 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_sample.h
+++ b/src/gallium/auxiliary/gallivm/lp_bld_sample.h
@@ -113,9 +113,9 @@ struct lp_sampler_dynamic_state
                   unsigned unit);
 
    LLVMValueRef
-   (*stride)( struct lp_sampler_dynamic_state *state,
-              LLVMBuilderRef builder,
-              unsigned unit);
+   (*row_stride)( struct lp_sampler_dynamic_state *state,
+                  LLVMBuilderRef builder,
+                  unsigned unit);
 
    LLVMValueRef
    (*data_ptr)( struct lp_sampler_dynamic_state *state,
diff --git a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
index 9358ad3860..2fc22fb1fd 100644
--- a/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
+++ b/src/gallium/auxiliary/gallivm/lp_bld_sample_soa.c
@@ -138,6 +138,34 @@ lp_build_get_const_mipmap_level(struct lp_build_sample_context *bld,
 }
 
 
+/**
+ * Dereference stride_array[mipmap_level] array to get a stride.
+ * Return stride as a vector.
+ */
+static LLVMValueRef
+lp_build_get_level_stride_vec(struct lp_build_sample_context *bld,
+                              LLVMValueRef stride_array, LLVMValueRef level)
+{
+   LLVMValueRef indexes[2], stride;
+   indexes[0] = LLVMConstInt(LLVMInt32Type(), 0, 0);
+   indexes[1] = level;
+   stride = LLVMBuildGEP(bld->builder, stride_array, indexes, 2, "");
+   stride = LLVMBuildLoad(bld->builder, stride, "");
+   stride = lp_build_broadcast_scalar(&bld->int_coord_bld, stride);
+   return stride;
+}
+
+
+/** Dereference stride_array[0] array to get a stride (as vector). */
+static LLVMValueRef
+lp_build_get_const_level_stride_vec(struct lp_build_sample_context *bld,
+                                    LLVMValueRef stride_array, int level)
+{
+   LLVMValueRef lvl = LLVMConstInt(LLVMInt32Type(), level, 0);
+   return lp_build_get_level_stride_vec(bld, stride_array, lvl);
+}
+
+
 static int
 texture_dims(enum pipe_texture_target tex)
 {
@@ -1190,7 +1218,7 @@ lp_build_sample_general(struct lp_build_sample_context *bld,
                         LLVMValueRef width_vec,
                         LLVMValueRef height_vec,
                         LLVMValueRef depth_vec,
-                        LLVMValueRef row_stride_vec,
+                        LLVMValueRef row_stride_array,
                         LLVMValueRef img_stride_vec,
                         LLVMValueRef data_array,
                         LLVMValueRef *colors_out)
@@ -1248,7 +1276,8 @@ lp_build_sample_general(struct lp_build_sample_context *bld,
    width0_vec = lp_build_minify(bld, width_vec, ilevel0_vec);
    if (dims >= 2) {
       height0_vec = lp_build_minify(bld, height_vec, ilevel0_vec);
-      row_stride0_vec = lp_build_minify(bld, row_stride_vec, ilevel0_vec);
+      row_stride0_vec = lp_build_get_level_stride_vec(bld, row_stride_array,
+                                                      ilevel0);
       if (dims == 3) {
          depth0_vec = lp_build_minify(bld, depth_vec, ilevel0_vec);
       }
@@ -1258,7 +1287,8 @@ lp_build_sample_general(struct lp_build_sample_context *bld,
       width1_vec = lp_build_minify(bld, width_vec, ilevel1_vec);
       if (dims >= 2) {
          height1_vec = lp_build_minify(bld, height_vec, ilevel1_vec);
-         row_stride1_vec = lp_build_minify(bld, row_stride_vec, ilevel1_vec);
+         row_stride1_vec = lp_build_get_level_stride_vec(bld, row_stride_array,
+                                                         ilevel1);
          if (dims == 3) {
             depth1_vec = lp_build_minify(bld, depth_vec, ilevel1_vec);
          }
@@ -1380,7 +1410,7 @@ lp_build_sample_2d_linear_aos(struct lp_build_sample_context *bld,
                               LLVMValueRef t,
                               LLVMValueRef width,
                               LLVMValueRef height,
-                              LLVMValueRef stride,
+                              LLVMValueRef stride_array,
                               LLVMValueRef data_array,
                               LLVMValueRef *texel)
 {
@@ -1397,6 +1427,7 @@ lp_build_sample_2d_linear_aos(struct lp_build_sample_context *bld,
    LLVMValueRef neighbors_hi[2][2];
    LLVMValueRef packed, packed_lo, packed_hi;
    LLVMValueRef unswizzled[4];
+   LLVMValueRef stride;
 
    lp_build_context_init(&i32, builder, lp_type_int_vec(32));
    lp_build_context_init(&h16, builder, lp_type_ufixed(16));
@@ -1504,6 +1535,8 @@ lp_build_sample_2d_linear_aos(struct lp_build_sample_context *bld,
       t_fpart_hi = LLVMBuildShuffleVector(builder, t_fpart, h16.undef, shuffle_hi, "");
    }
 
+   stride = lp_build_get_const_level_stride_vec(bld, stride_array, 0);
+
    /*
     * Fetch the pixels as 4 x 32bit (rgba order might differ):
     *
@@ -1628,7 +1661,7 @@ lp_build_sample_soa(LLVMBuilderRef builder,
    LLVMValueRef width, width_vec;
    LLVMValueRef height, height_vec;
    LLVMValueRef depth, depth_vec;
-   LLVMValueRef stride, stride_vec;
+   LLVMValueRef stride_array;
    LLVMValueRef data_array;
    LLVMValueRef s;
    LLVMValueRef t;
@@ -1664,7 +1697,7 @@ lp_build_sample_soa(LLVMBuilderRef builder,
    width = dynamic_state->width(dynamic_state, builder, unit);
    height = dynamic_state->height(dynamic_state, builder, unit);
    depth = dynamic_state->depth(dynamic_state, builder, unit);
-   stride = dynamic_state->stride(dynamic_state, builder, unit);
+   stride_array = dynamic_state->row_stride(dynamic_state, builder, unit);
    data_array = dynamic_state->data_ptr(dynamic_state, builder, unit);
    /* Note that data_array is an array[level] of pointers to texture images */
 
@@ -1675,7 +1708,6 @@ lp_build_sample_soa(LLVMBuilderRef builder,
    width_vec = lp_build_broadcast_scalar(&bld.uint_coord_bld, width);
    height_vec = lp_build_broadcast_scalar(&bld.uint_coord_bld, height);
    depth_vec = lp_build_broadcast_scalar(&bld.uint_coord_bld, depth);
-   stride_vec = lp_build_broadcast_scalar(&bld.uint_coord_bld, stride);
 
    if (lp_format_is_rgba8(bld.format_desc) &&
        static_state->min_img_filter == PIPE_TEX_FILTER_LINEAR &&
@@ -1685,13 +1717,13 @@ lp_build_sample_soa(LLVMBuilderRef builder,
        is_simple_wrap_mode(static_state->wrap_t)) {
       /* special case */
       lp_build_sample_2d_linear_aos(&bld, s, t, width_vec, height_vec,
-                                    stride_vec, data_array, texel);
+                                    stride_array, data_array, texel);
    }
    else {
       lp_build_sample_general(&bld, unit, s, t, r,
                               width, height, depth,
                               width_vec, height_vec, depth_vec,
-                              stride_vec, NULL, data_array,
+                              stride_array, NULL, data_array,
                               texel);
    }
 
diff --git a/src/gallium/drivers/llvmpipe/lp_jit.c b/src/gallium/drivers/llvmpipe/lp_jit.c
index 08c8f93794..5887613120 100644
--- a/src/gallium/drivers/llvmpipe/lp_jit.c
+++ b/src/gallium/drivers/llvmpipe/lp_jit.c
@@ -57,7 +57,8 @@ lp_jit_init_globals(struct llvmpipe_screen *screen)
       elem_types[LP_JIT_TEXTURE_HEIGHT] = LLVMInt32Type();
       elem_types[LP_JIT_TEXTURE_DEPTH] = LLVMInt32Type();
       elem_types[LP_JIT_TEXTURE_LAST_LEVEL] = LLVMInt32Type();
-      elem_types[LP_JIT_TEXTURE_STRIDE] = LLVMInt32Type();
+      elem_types[LP_JIT_TEXTURE_ROW_STRIDE] =
+         LLVMArrayType(LLVMInt32Type(), LP_MAX_TEXTURE_2D_LEVELS);
       elem_types[LP_JIT_TEXTURE_DATA] =
          LLVMArrayType(LLVMPointerType(LLVMInt8Type(), 0),
                        LP_MAX_TEXTURE_2D_LEVELS);
@@ -76,9 +77,9 @@ lp_jit_init_globals(struct llvmpipe_screen *screen)
       LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, last_level,
                              screen->target, texture_type,
                              LP_JIT_TEXTURE_LAST_LEVEL);
-      LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, stride,
+      LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, row_stride,
                              screen->target, texture_type,
-                             LP_JIT_TEXTURE_STRIDE);
+                             LP_JIT_TEXTURE_ROW_STRIDE);
       LP_CHECK_MEMBER_OFFSET(struct lp_jit_texture, data,
                              screen->target, texture_type,
                              LP_JIT_TEXTURE_DATA);
diff --git a/src/gallium/drivers/llvmpipe/lp_jit.h b/src/gallium/drivers/llvmpipe/lp_jit.h
index 5cc7a12c03..13167ae3bf 100644
--- a/src/gallium/drivers/llvmpipe/lp_jit.h
+++ b/src/gallium/drivers/llvmpipe/lp_jit.h
@@ -51,7 +51,7 @@ struct lp_jit_texture
    uint32_t height;
    uint32_t depth;
    uint32_t last_level;
-   uint32_t stride;
+   uint32_t row_stride[LP_MAX_TEXTURE_2D_LEVELS];
    const void *data[LP_MAX_TEXTURE_2D_LEVELS];
 };
 
@@ -61,7 +61,7 @@ enum {
    LP_JIT_TEXTURE_HEIGHT,
    LP_JIT_TEXTURE_DEPTH,
    LP_JIT_TEXTURE_LAST_LEVEL,
-   LP_JIT_TEXTURE_STRIDE,
+   LP_JIT_TEXTURE_ROW_STRIDE,
    LP_JIT_TEXTURE_DATA
 };
 
diff --git a/src/gallium/drivers/llvmpipe/lp_setup.c b/src/gallium/drivers/llvmpipe/lp_setup.c
index d6d37c4809..c870f89d01 100644
--- a/src/gallium/drivers/llvmpipe/lp_setup.c
+++ b/src/gallium/drivers/llvmpipe/lp_setup.c
@@ -471,13 +471,13 @@ lp_setup_set_sampler_textures( struct setup_context *setup,
          jit_tex->height = tex->height0;
          jit_tex->depth = tex->depth0;
          jit_tex->last_level = tex->last_level;
-         jit_tex->stride = lp_tex->stride[0];
          if (!lp_tex->dt) {
             /* regular texture - setup array of mipmap level pointers */
             int j;
             for (j = 0; j <= tex->last_level; j++) {
                jit_tex->data[j] =
                   (ubyte *) lp_tex->data + lp_tex->level_offset[j];
+               jit_tex->row_stride[j] = lp_tex->stride[j];
             }
          }
          else {
@@ -490,6 +490,7 @@ lp_setup_set_sampler_textures( struct setup_context *setup,
             struct sw_winsys *winsys = screen->winsys;
             jit_tex->data[0] = winsys->displaytarget_map(winsys, lp_tex->dt,
                                                       PIPE_BUFFER_USAGE_CPU_READ);
+            jit_tex->row_stride[0] = lp_tex->stride[0];
             assert(jit_tex->data[0]);
          }
 
diff --git a/src/gallium/drivers/llvmpipe/lp_tex_sample_llvm.c b/src/gallium/drivers/llvmpipe/lp_tex_sample_llvm.c
index 5a3cf37d6d..662508af61 100644
--- a/src/gallium/drivers/llvmpipe/lp_tex_sample_llvm.c
+++ b/src/gallium/drivers/llvmpipe/lp_tex_sample_llvm.c
@@ -147,7 +147,7 @@ LP_LLVM_TEXTURE_MEMBER(width,      LP_JIT_TEXTURE_WIDTH, TRUE)
 LP_LLVM_TEXTURE_MEMBER(height,     LP_JIT_TEXTURE_HEIGHT, TRUE)
 LP_LLVM_TEXTURE_MEMBER(depth,      LP_JIT_TEXTURE_DEPTH, TRUE)
 LP_LLVM_TEXTURE_MEMBER(last_level, LP_JIT_TEXTURE_LAST_LEVEL, TRUE)
-LP_LLVM_TEXTURE_MEMBER(stride,     LP_JIT_TEXTURE_STRIDE, TRUE)
+LP_LLVM_TEXTURE_MEMBER(row_stride, LP_JIT_TEXTURE_ROW_STRIDE, FALSE)
 LP_LLVM_TEXTURE_MEMBER(data_ptr,   LP_JIT_TEXTURE_DATA, FALSE)
 
 
@@ -204,7 +204,7 @@ lp_llvm_sampler_soa_create(const struct lp_sampler_static_state *static_state,
    sampler->dynamic_state.base.height = lp_llvm_texture_height;
    sampler->dynamic_state.base.depth = lp_llvm_texture_depth;
    sampler->dynamic_state.base.last_level = lp_llvm_texture_last_level;
-   sampler->dynamic_state.base.stride = lp_llvm_texture_stride;
+   sampler->dynamic_state.base.row_stride = lp_llvm_texture_row_stride;
    sampler->dynamic_state.base.data_ptr = lp_llvm_texture_data_ptr;
    sampler->dynamic_state.static_state = static_state;
    sampler->dynamic_state.context_ptr = context_ptr;