From: Nicolai Hähnle <nicolai.haeh...@amd.com>
... by straight-forward search & replace, and eliminate
emit_llvm_intrinsic.
---
src/amd/common/ac_nir_to_llvm.c | 120 ++++++++++++++--------------------------
1 file changed, 41 insertions(+), 79 deletions(-)
diff --git a/src/amd/common/ac_nir_to_llvm.c b/src/amd/common/ac_nir_to_llvm.c
index 0fd8559..fefd5e7 100644
--- a/src/amd/common/ac_nir_to_llvm.c
+++ b/src/amd/common/ac_nir_to_llvm.c
@@ -135,24 +135,20 @@ struct nir_to_llvm_context {
bool has_ds_bpermute;
};
struct ac_tex_info {
LLVMValueRef args[12];
int arg_count;
LLVMTypeRef dst_type;
bool has_offset;
};
-static LLVMValueRef
-emit_llvm_intrinsic(struct nir_to_llvm_context *ctx, const char *name,
- LLVMTypeRef return_type, LLVMValueRef *params,
- unsigned param_count, unsigned attr_mask);
static LLVMValueRef get_sampler_desc(struct nir_to_llvm_context *ctx,
nir_deref_var *deref,
enum desc_type desc_type);
static unsigned radeon_llvm_reg_index_soa(unsigned index, unsigned chan)
{
return (index * 4) + chan;
}
static unsigned llvm_get_type_size(LLVMTypeRef type)
{
@@ -790,44 +786,44 @@ static LLVMValueRef emit_float_cmp(struct
nir_to_llvm_context *ctx,
LLVMConstInt(ctx->i32, 0, false), "");
}
static LLVMValueRef emit_intrin_1f_param(struct nir_to_llvm_context *ctx,
const char *intrin,
LLVMValueRef src0)
{
LLVMValueRef params[] = {
to_float(ctx, src0),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 1,
AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 1,
AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_intrin_2f_param(struct nir_to_llvm_context *ctx,
const char *intrin,
LLVMValueRef src0, LLVMValueRef src1)
{
LLVMValueRef params[] = {
to_float(ctx, src0),
to_float(ctx, src1),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 2,
AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 2,
AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_intrin_3f_param(struct nir_to_llvm_context *ctx,
const char *intrin,
LLVMValueRef src0, LLVMValueRef src1,
LLVMValueRef src2)
{
LLVMValueRef params[] = {
to_float(ctx, src0),
to_float(ctx, src1),
to_float(ctx, src2),
};
- return emit_llvm_intrinsic(ctx, intrin, ctx->f32, params, 3,
AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->f32, params, 3,
AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_bcsel(struct nir_to_llvm_context *ctx,
LLVMValueRef src0, LLVMValueRef src1,
LLVMValueRef src2)
{
LLVMValueRef v = LLVMBuildICmp(ctx->builder, LLVMIntNE, src0,
ctx->i32zero, "");
return LLVMBuildSelect(ctx->builder, v, src1, src2, "");
}
@@ -839,27 +835,27 @@ static LLVMValueRef emit_find_lsb(struct
nir_to_llvm_context *ctx,
/* The value of 1 means that ffs(x=0) = undef, so LLVM won't
* add special code to check for x=0. The reason is that
* the LLVM behavior for x=0 is different from what we
* need here.
*
* The hardware already implements the correct behavior.
*/
LLVMConstInt(ctx->i32, 1, false),
};
- return emit_llvm_intrinsic(ctx, "llvm.cttz.i32", ctx->i32, params, 2,
AC_FUNC_ATTR_READNONE);
+ return ac_emit_llvm_intrinsic(&ctx->ac, "llvm.cttz.i32", ctx->i32,
params, 2, AC_FUNC_ATTR_READNONE);
}
static LLVMValueRef emit_ifind_msb(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
- LLVMValueRef msb = emit_llvm_intrinsic(ctx, "llvm.AMDGPU.flbit.i32",
+ LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.AMDGPU.flbit.i32",
ctx->i32, &src0, 1,
AC_FUNC_ATTR_READNONE);
/* The HW returns the last bit index from MSB, but NIR wants
* the index from LSB. Invert it by doing "31 - msb". */
msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
msb, "");
LLVMValueRef all_ones = LLVMConstInt(ctx->i32, -1, true);
LLVMValueRef cond = LLVMBuildOr(ctx->builder,
@@ -871,21 +867,21 @@ static LLVMValueRef emit_ifind_msb(struct
nir_to_llvm_context *ctx,
return LLVMBuildSelect(ctx->builder, cond, all_ones, msb, "");
}
static LLVMValueRef emit_ufind_msb(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
LLVMValueRef args[2] = {
src0,
ctx->i32one,
};
- LLVMValueRef msb = emit_llvm_intrinsic(ctx, "llvm.ctlz.i32",
+ LLVMValueRef msb = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctlz.i32",
ctx->i32, args, ARRAY_SIZE(args),
AC_FUNC_ATTR_READNONE);
/* The HW returns the last bit index from MSB, but NIR wants
* the index from LSB. Invert it by doing "31 - msb". */
msb = LLVMBuildSub(ctx->builder, LLVMConstInt(ctx->i32, 31, false),
msb, "");
return LLVMBuildSelect(ctx->builder,
LLVMBuildICmp(ctx->builder, LLVMIntEQ, src0,
@@ -935,38 +931,38 @@ static LLVMValueRef emit_isign(struct nir_to_llvm_context
*ctx,
}
static LLVMValueRef emit_ffract(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
const char *intr = "llvm.floor.f32";
LLVMValueRef fsrc0 = to_float(ctx, src0);
LLVMValueRef params[] = {
fsrc0,
};
- LLVMValueRef floor = emit_llvm_intrinsic(ctx, intr,
+ LLVMValueRef floor = ac_emit_llvm_intrinsic(&ctx->ac, intr,
ctx->f32, params, 1,
AC_FUNC_ATTR_READNONE);
return LLVMBuildFSub(ctx->builder, fsrc0, floor, "");
}
static LLVMValueRef emit_uint_carry(struct nir_to_llvm_context *ctx,
const char *intrin,
LLVMValueRef src0, LLVMValueRef src1)
{
LLVMTypeRef ret_type;
LLVMTypeRef types[] = { ctx->i32, ctx->i1 };
LLVMValueRef res;
LLVMValueRef params[] = { src0, src1 };
ret_type = LLVMStructTypeInContext(ctx->context, types,
2, true);
- res = emit_llvm_intrinsic(ctx, intrin, ret_type,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ret_type,
params, 2, AC_FUNC_ATTR_READNONE);
res = LLVMBuildExtractValue(ctx->builder, res, 1, "");
res = LLVMBuildZExt(ctx->builder, res, ctx->i32, "");
return res;
}
static LLVMValueRef emit_b2f(struct nir_to_llvm_context *ctx,
LLVMValueRef src0)
{
@@ -998,21 +994,21 @@ static LLVMValueRef emit_imul_high(struct
nir_to_llvm_context *ctx,
result = LLVMBuildTrunc(ctx->builder, dst64, ctx->i32, "");
return result;
}
static LLVMValueRef emit_bitfield_extract(struct nir_to_llvm_context *ctx,
const char *intrin,
LLVMValueRef srcs[3])
{
LLVMValueRef result;
LLVMValueRef icond = LLVMBuildICmp(ctx->builder, LLVMIntEQ, srcs[2],
LLVMConstInt(ctx->i32, 32, false), "");
- result = emit_llvm_intrinsic(ctx, intrin, ctx->i32, srcs, 3,
AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, intrin, ctx->i32, srcs, 3,
AC_FUNC_ATTR_READNONE);
result = LLVMBuildSelect(ctx->builder, icond, srcs[0], result, "");
return result;
}
static LLVMValueRef emit_bitfield_insert(struct nir_to_llvm_context *ctx,
LLVMValueRef src0, LLVMValueRef src1,
LLVMValueRef src2, LLVMValueRef src3)
{
LLVMValueRef bfi_args[3], result;
@@ -1103,25 +1099,25 @@ static void set_range_metadata(struct
nir_to_llvm_context *ctx,
range_md = LLVMMDNodeInContext(context, md_args, 2);
LLVMSetMetadata(value, ctx->range_md_kind, range_md);
}
static LLVMValueRef get_thread_id(struct nir_to_llvm_context *ctx)
{
LLVMValueRef tid;
LLVMValueRef tid_args[2];
tid_args[0] = LLVMConstInt(ctx->i32, 0xffffffff, false);
tid_args[1] = ctx->i32zero;
- tid_args[1] = emit_llvm_intrinsic(ctx,
+ tid_args[1] = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.mbcnt.lo", ctx->i32,
tid_args, 2, AC_FUNC_ATTR_READNONE);
- tid = emit_llvm_intrinsic(ctx,
+ tid = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.mbcnt.hi", ctx->i32,
tid_args, 2, AC_FUNC_ATTR_READNONE);
set_range_metadata(ctx, tid, 0, 64);
return tid;
}
/*
* SI implements derivatives using the local data store (LDS)
* All writes to the LDS happen in all executing threads at
* the same time. TID is the Thread ID for the current
@@ -1184,27 +1180,27 @@ static LLVMValueRef emit_ddxy(struct
nir_to_llvm_context *ctx,
else
idx = 2;
trbl_tid = LLVMBuildAdd(ctx->builder, tl_tid,
LLVMConstInt(ctx->i32, idx, false), "");
if (ctx->has_ds_bpermute) {
args[0] = LLVMBuildMul(ctx->builder, tl_tid,
LLVMConstInt(ctx->i32, 4, false), "");
args[1] = src0;
- tl = emit_llvm_intrinsic(ctx, "llvm.amdgcn.ds.bpermute",
+ tl = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.ds.bpermute",
ctx->i32, args, 2,
AC_FUNC_ATTR_READNONE);
args[0] = LLVMBuildMul(ctx->builder, trbl_tid,
LLVMConstInt(ctx->i32, 4, false), "");
- trbl = emit_llvm_intrinsic(ctx, "llvm.amdgcn.ds.bpermute",
+ trbl = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.ds.bpermute",
ctx->i32, args, 2,
AC_FUNC_ATTR_READNONE);
} else {
LLVMValueRef store_ptr, load_ptr0, load_ptr1;
store_ptr = build_gep0(ctx, ctx->lds, thread_id);
load_ptr0 = build_gep0(ctx, ctx->lds, tl_tid);
load_ptr1 = build_gep0(ctx, ctx->lds, trbl_tid);
LLVMBuildStore(ctx->builder, src0, store_ptr);
@@ -1470,24 +1466,24 @@ static void visit_alu(struct nir_to_llvm_context *ctx,
nir_alu_instr *instr)
case nir_op_ibitfield_extract:
result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.i32", src);
break;
case nir_op_ubitfield_extract:
result = emit_bitfield_extract(ctx, "llvm.AMDGPU.bfe.u32", src);
break;
case nir_op_bitfield_insert:
result = emit_bitfield_insert(ctx, src[0], src[1], src[2],
src[3]);
break;
case nir_op_bitfield_reverse:
- result = emit_llvm_intrinsic(ctx, "llvm.bitreverse.i32",
ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.bitreverse.i32", ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
break;
case nir_op_bit_count:
- result = emit_llvm_intrinsic(ctx, "llvm.ctpop.i32", ctx->i32,
src, 1, AC_FUNC_ATTR_READNONE);
+ result = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.ctpop.i32",
ctx->i32, src, 1, AC_FUNC_ATTR_READNONE);
break;
case nir_op_vec2:
case nir_op_vec3:
case nir_op_vec4:
for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs;
i++)
src[i] = to_integer(ctx, src[i]);
result = build_gather_values(ctx, src, num_components);
break;
case nir_op_f2i:
src[0] = to_float(ctx, src[0]);
@@ -1599,54 +1595,20 @@ static void visit_load_const(struct nir_to_llvm_context
*ctx,
static LLVMValueRef cast_ptr(struct nir_to_llvm_context *ctx, LLVMValueRef ptr,
LLVMTypeRef type)
{
int addr_space = LLVMGetPointerAddressSpace(LLVMTypeOf(ptr));
return LLVMBuildBitCast(ctx->builder, ptr,
LLVMPointerType(type, addr_space), "");
}
static LLVMValueRef
-emit_llvm_intrinsic(struct nir_to_llvm_context *ctx, const char *name,
- LLVMTypeRef return_type, LLVMValueRef *params,
- unsigned param_count, unsigned attrib_mask)
-{
- LLVMValueRef function;
-
- function = LLVMGetNamedFunction(ctx->module, name);
- if (!function) {
- LLVMTypeRef param_types[32], function_type;
- unsigned i;
-
- assert(param_count <= 32);
-
- for (i = 0; i < param_count; ++i) {
- assert(params[i]);
- param_types[i] = LLVMTypeOf(params[i]);
- }
- function_type =
- LLVMFunctionType(return_type, param_types, param_count, 0);
- function = LLVMAddFunction(ctx->module, name, function_type);
-
- LLVMSetFunctionCallConv(function, LLVMCCallConv);
- LLVMSetLinkage(function, LLVMExternalLinkage);
-
- attrib_mask |= AC_FUNC_ATTR_NOUNWIND;
- while (attrib_mask) {
- enum ac_func_attr attr = 1u << u_bit_scan(&attrib_mask);
- ac_add_function_attr(function, -1, attr);
- }
- }
- return LLVMBuildCall(ctx->builder, function, params, param_count, "");
-}
-
-static LLVMValueRef
get_buffer_size(struct nir_to_llvm_context *ctx, LLVMValueRef descriptor, bool
in_elements)
{
LLVMValueRef size =
LLVMBuildExtractElement(ctx->builder, descriptor,
LLVMConstInt(ctx->i32, 2, false), "");
/* VI only */
if (ctx->options->chip_class >= VI && in_elements) {
/* On VI, the descriptor contains the size in bytes,
* but TXQ must return the size in elements.
@@ -1701,21 +1663,21 @@ static LLVMValueRef radv_lower_gather4_integer(struct
nir_to_llvm_context *ctx,
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, false);
txq_args[txq_arg_count++] = tinfo->args[1];
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0xf, 0); /*
dmask */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
unorm */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
r128 */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, da ? 1 : 0,
0);
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
glc */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
slc */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
tfe */
txq_args[txq_arg_count++] = LLVMConstInt(ctx->i32, 0, 0); /*
lwe */
- size = emit_llvm_intrinsic(ctx, "llvm.SI.getresinfo.i32",
ctx->v4i32,
+ size = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.getresinfo.i32", ctx->v4i32,
txq_args, txq_arg_count,
AC_FUNC_ATTR_READNONE);
for (c = 0; c < 2; c++) {
half_texel[c] = LLVMBuildExtractElement(ctx->builder,
size,
LLVMConstInt(ctx->i32, c, false), "");
half_texel[c] = LLVMBuildUIToFP(ctx->builder,
half_texel[c], ctx->f32, "");
half_texel[c] = emit_fdiv(ctx, ctx->f32one,
half_texel[c]);
half_texel[c] = LLVMBuildFMul(ctx->builder,
half_texel[c],
LLVMConstReal(ctx->f32,
-0.5), "");
@@ -1726,21 +1688,21 @@ static LLVMValueRef radv_lower_gather4_integer(struct
nir_to_llvm_context *ctx,
LLVMValueRef tmp;
LLVMValueRef index = LLVMConstInt(ctx->i32, coord_vgpr_index +
c, 0);
tmp = LLVMBuildExtractElement(ctx->builder, coord, index, "");
tmp = LLVMBuildBitCast(ctx->builder, tmp, ctx->f32, "");
tmp = LLVMBuildFAdd(ctx->builder, tmp, half_texel[c], "");
tmp = LLVMBuildBitCast(ctx->builder, tmp, ctx->i32, "");
coord = LLVMBuildInsertElement(ctx->builder, coord, tmp, index,
"");
}
tinfo->args[0] = coord;
- return emit_llvm_intrinsic(ctx, intr_name, tinfo->dst_type,
tinfo->args, tinfo->arg_count,
+ return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type,
tinfo->args, tinfo->arg_count,
AC_FUNC_ATTR_READNONE |
AC_FUNC_ATTR_NOUNWIND);
}
static LLVMValueRef build_tex_intrinsic(struct nir_to_llvm_context *ctx,
nir_tex_instr *instr,
struct ac_tex_info *tinfo)
{
const char *name = "llvm.SI.image.sample";
const char *infix = "";
@@ -1794,21 +1756,21 @@ static LLVMValueRef build_tex_intrinsic(struct
nir_to_llvm_context *ctx,
sprintf(intr_name, "%s%s%s%s.%s", name, is_shadow ? ".c" : "", infix,
has_offset ? ".o" : "", type);
if (instr->op == nir_texop_tg4) {
enum glsl_base_type stype =
glsl_get_sampler_result_type(instr->texture->var->type);
if (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT) {
return radv_lower_gather4_integer(ctx, tinfo, instr,
intr_name,
(int)has_offset +
(int)is_shadow);
}
}
- return emit_llvm_intrinsic(ctx, intr_name, tinfo->dst_type,
tinfo->args, tinfo->arg_count,
+ return ac_emit_llvm_intrinsic(&ctx->ac, intr_name, tinfo->dst_type,
tinfo->args, tinfo->arg_count,
AC_FUNC_ATTR_READNONE |
AC_FUNC_ATTR_NOUNWIND);
}
static LLVMValueRef visit_vulkan_resource_index(struct nir_to_llvm_context
*ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef index = get_src(ctx, instr->src[0]);
unsigned desc_set = nir_intrinsic_desc_set(instr);
unsigned binding = nir_intrinsic_binding(instr);
@@ -1918,21 +1880,21 @@ static void visit_store_ssbo(struct nir_to_llvm_context
*ctx,
data = base_data;
store_name = "llvm.amdgcn.buffer.store.f32";
}
offset = base_offset;
if (start != 0) {
offset = LLVMBuildAdd(ctx->builder, offset,
LLVMConstInt(ctx->i32, start * 4, false), "");
}
params[0] = data;
params[3] = offset;
- emit_llvm_intrinsic(ctx, store_name,
+ ac_emit_llvm_intrinsic(&ctx->ac, store_name,
LLVMVoidTypeInContext(ctx->context),
params, 6, 0);
}
}
static LLVMValueRef visit_atomic_ssbo(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
const char *name;
LLVMValueRef params[6];
int arg_count = 0;
@@ -1976,21 +1938,21 @@ static LLVMValueRef visit_atomic_ssbo(struct
nir_to_llvm_context *ctx,
case nir_intrinsic_ssbo_atomic_exchange:
name = "llvm.amdgcn.buffer.atomic.swap";
break;
case nir_intrinsic_ssbo_atomic_comp_swap:
name = "llvm.amdgcn.buffer.atomic.cmpswap";
break;
default:
abort();
}
- return emit_llvm_intrinsic(ctx, name, ctx->i32, params, arg_count, 0);
+ return ac_emit_llvm_intrinsic(&ctx->ac, name, ctx->i32, params,
arg_count, 0);
}
static LLVMValueRef visit_load_buffer(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
const char *load_name;
LLVMTypeRef data_type = ctx->f32;
if (instr->num_components == 3)
data_type = LLVMVectorType(ctx->f32, 4);
else if (instr->num_components > 1)
@@ -2007,21 +1969,21 @@ static LLVMValueRef visit_load_buffer(struct
nir_to_llvm_context *ctx,
LLVMValueRef params[] = {
get_src(ctx, instr->src[0]),
LLVMConstInt(ctx->i32, 0, false),
get_src(ctx, instr->src[1]),
LLVMConstInt(ctx->i1, 0, false),
LLVMConstInt(ctx->i1, 0, false),
};
LLVMValueRef ret =
- emit_llvm_intrinsic(ctx, load_name, data_type, params, 5, 0);
+ ac_emit_llvm_intrinsic(&ctx->ac, load_name, data_type, params, 5,
0);
if (instr->num_components == 3)
ret = trim_vector(ctx, ret, 3);
return LLVMBuildBitCast(ctx->builder, ret,
get_def_type(ctx, &instr->dest.ssa), "");
}
static LLVMValueRef visit_load_ubo_buffer(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
@@ -2031,21 +1993,21 @@ static LLVMValueRef visit_load_ubo_buffer(struct
nir_to_llvm_context *ctx,
LLVMValueRef offset = get_src(ctx, instr->src[1]);
rsrc = LLVMBuildBitCast(ctx->builder, rsrc, LLVMVectorType(ctx->i8,
16), "");
for (unsigned i = 0; i < instr->num_components; ++i) {
LLVMValueRef params[] = {
rsrc,
LLVMBuildAdd(ctx->builder, LLVMConstInt(ctx->i32, 4 *
i, 0),
offset, "")
};
- results[i] = emit_llvm_intrinsic(ctx, "llvm.SI.load.const",
ctx->f32,
+ results[i] = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.load.const", ctx->f32,
params, 2,
AC_FUNC_ATTR_READNONE);
}
ret = build_gather_values(ctx, results, instr->num_components);
return LLVMBuildBitCast(ctx->builder, ret,
get_def_type(ctx, &instr->dest.ssa), "");
}
static void
@@ -2443,21 +2405,21 @@ static LLVMValueRef visit_image_load(struct
nir_to_llvm_context *ctx,
type = instr->variables[0]->deref.child->type;
type = glsl_without_array(type);
if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
params[0] = get_sampler_desc(ctx, instr->variables[0],
DESC_BUFFER);
params[1] = LLVMBuildExtractElement(ctx->builder, get_src(ctx,
instr->src[0]),
LLVMConstInt(ctx->i32, 0,
false), ""); /* vindex */
params[2] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
params[3] = LLVMConstInt(ctx->i1, 0, false); /* glc */
params[4] = LLVMConstInt(ctx->i1, 0, false); /* slc */
- res = emit_llvm_intrinsic(ctx,
"llvm.amdgcn.buffer.load.format.v4f32", ctx->v4f32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.buffer.load.format.v4f32", ctx->v4f32,
params, 5, 0);
res = trim_vector(ctx, res, instr->dest.ssa.num_components);
res = to_integer(ctx, res);
} else {
bool is_da = glsl_sampler_type_is_array(type) ||
glsl_get_sampler_dim(type) ==
GLSL_SAMPLER_DIM_CUBE;
bool add_frag_pos = glsl_get_sampler_dim(type) ==
GLSL_SAMPLER_DIM_SUBPASS;
LLVMValueRef da = is_da ? ctx->i32one : ctx->i32zero;
LLVMValueRef glc = LLVMConstInt(ctx->i1, 0, false);
@@ -2478,21 +2440,21 @@ static LLVMValueRef visit_image_load(struct
nir_to_llvm_context *ctx,
params[5] = lwe;
params[6] = da;
}
get_image_intr_name("llvm.amdgcn.image.load",
ctx->v4f32, /* vdata */
LLVMTypeOf(params[0]), /* coords */
LLVMTypeOf(params[1]), /* rsrc */
intrinsic_name, sizeof(intrinsic_name));
- res = emit_llvm_intrinsic(ctx, intrinsic_name, ctx->v4f32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name,
ctx->v4f32,
params, 7, AC_FUNC_ATTR_READONLY);
}
return to_integer(ctx, res);
}
static void visit_image_store(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef params[8];
char intrinsic_name[64];
@@ -2505,21 +2467,21 @@ static void visit_image_store(struct
nir_to_llvm_context *ctx,
ctx->shader_info->fs.writes_memory = true;
if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
params[0] = to_float(ctx, get_src(ctx, instr->src[2])); /* data
*/
params[1] = get_sampler_desc(ctx, instr->variables[0],
DESC_BUFFER);
params[2] = LLVMBuildExtractElement(ctx->builder, get_src(ctx,
instr->src[0]),
LLVMConstInt(ctx->i32, 0,
false), ""); /* vindex */
params[3] = LLVMConstInt(ctx->i32, 0, false); /* voffset */
params[4] = i1false; /* glc */
params[5] = i1false; /* slc */
- emit_llvm_intrinsic(ctx,
"llvm.amdgcn.buffer.store.format.v4f32", ctx->voidt,
+ ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.amdgcn.buffer.store.format.v4f32", ctx->voidt,
params, 6, 0);
} else {
bool is_da = glsl_sampler_type_is_array(type) ||
glsl_get_sampler_dim(type) ==
GLSL_SAMPLER_DIM_CUBE;
LLVMValueRef da = is_da ? i1true : i1false;
LLVMValueRef glc = i1false;
LLVMValueRef slc = i1false;
params[0] = to_float(ctx, get_src(ctx, instr->src[2]));
params[1] = get_image_coords(ctx, instr, false); /* coords */
@@ -2537,21 +2499,21 @@ static void visit_image_store(struct
nir_to_llvm_context *ctx,
params[6] = lwe;
params[7] = da;
}
get_image_intr_name("llvm.amdgcn.image.store",
LLVMTypeOf(params[0]), /* vdata */
LLVMTypeOf(params[1]), /* coords */
LLVMTypeOf(params[2]), /* rsrc */
intrinsic_name, sizeof(intrinsic_name));
- emit_llvm_intrinsic(ctx, intrinsic_name, ctx->voidt,
+ ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->voidt,
params, 8, 0);
}
}
static LLVMValueRef visit_image_atomic(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef params[6];
int param_count = 0;
@@ -2615,21 +2577,21 @@ static LLVMValueRef visit_image_atomic(struct
nir_to_llvm_context *ctx,
atomic_name = "cmpswap";
break;
default:
abort();
}
build_int_type_name(LLVMTypeOf(coords),
coords_type, sizeof(coords_type));
snprintf(intrinsic_name, sizeof(intrinsic_name),
"%s.%s.%s", base_name, atomic_name, coords_type);
- return emit_llvm_intrinsic(ctx, intrinsic_name, ctx->i32, params,
param_count, 0);
+ return ac_emit_llvm_intrinsic(&ctx->ac, intrinsic_name, ctx->i32,
params, param_count, 0);
}
static LLVMValueRef visit_image_size(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef res;
LLVMValueRef params[10];
const nir_variable *var = instr->variables[0]->var;
const struct glsl_type *type = instr->variables[0]->var->type;
bool da = glsl_sampler_type_is_array(var->type) ||
@@ -2643,64 +2605,64 @@ static LLVMValueRef visit_image_size(struct
nir_to_llvm_context *ctx,
params[1] = get_sampler_desc(ctx, instr->variables[0], DESC_IMAGE);
params[2] = LLVMConstInt(ctx->i32, 15, false);
params[3] = ctx->i32zero;
params[4] = ctx->i32zero;
params[5] = da ? ctx->i32one : ctx->i32zero;
params[6] = ctx->i32zero;
params[7] = ctx->i32zero;
params[8] = ctx->i32zero;
params[9] = ctx->i32zero;
- res = emit_llvm_intrinsic(ctx, "llvm.SI.getresinfo.i32", ctx->v4i32,
+ res = ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.getresinfo.i32",
ctx->v4i32,
params, 10, AC_FUNC_ATTR_READNONE);
if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
glsl_sampler_type_is_array(type)) {
LLVMValueRef two = LLVMConstInt(ctx->i32, 2, false);
LLVMValueRef six = LLVMConstInt(ctx->i32, 6, false);
LLVMValueRef z = LLVMBuildExtractElement(ctx->builder, res,
two, "");
z = LLVMBuildSDiv(ctx->builder, z, six, "");
res = LLVMBuildInsertElement(ctx->builder, res, z, two, "");
}
return res;
}
static void emit_waitcnt(struct nir_to_llvm_context *ctx)
{
LLVMValueRef args[1] = {
LLVMConstInt(ctx->i32, 0xf70, false),
};
- emit_llvm_intrinsic(ctx, "llvm.amdgcn.s.waitcnt",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.waitcnt",
ctx->voidt, args, 1, 0);
}
static void emit_barrier(struct nir_to_llvm_context *ctx)
{
// TODO tess
- emit_llvm_intrinsic(ctx, "llvm.amdgcn.s.barrier",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.amdgcn.s.barrier",
ctx->voidt, NULL, 0, 0);
}
static void emit_discard_if(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef cond;
ctx->shader_info->fs.can_discard = true;
cond = LLVMBuildICmp(ctx->builder, LLVMIntNE,
get_src(ctx, instr->src[0]),
ctx->i32zero, "");
cond = LLVMBuildSelect(ctx->builder, cond,
LLVMConstReal(ctx->f32, -1.0f),
ctx->f32zero, "");
- emit_llvm_intrinsic(ctx, "llvm.AMDGPU.kill",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kill",
LLVMVoidTypeInContext(ctx->context),
&cond, 1, 0);
}
static LLVMValueRef
visit_load_local_invocation_index(struct nir_to_llvm_context *ctx)
{
LLVMValueRef result;
LLVMValueRef thread_id = get_thread_id(ctx);
result = LLVMBuildAnd(ctx->builder, ctx->tg_size,
@@ -2902,21 +2864,21 @@ static LLVMValueRef visit_interp(struct
nir_to_llvm_context *ctx,
}
intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
for (chan = 0; chan < 2; chan++) {
LLVMValueRef args[4];
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
args[0] = llvm_chan;
args[1] = attr_number;
args[2] = ctx->prim_mask;
args[3] = interp_param;
- result[chan] = emit_llvm_intrinsic(ctx, intr_name,
+ result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
ctx->f32, args, args[3] ? 4
: 3,
AC_FUNC_ATTR_READNONE);
}
return build_gather_values(ctx, result, 2);
}
static void visit_intrinsic(struct nir_to_llvm_context *ctx,
nir_intrinsic_instr *instr)
{
LLVMValueRef result = NULL;
@@ -3013,21 +2975,21 @@ static void visit_intrinsic(struct nir_to_llvm_context
*ctx,
case nir_intrinsic_image_atomic_xor:
case nir_intrinsic_image_atomic_exchange:
case nir_intrinsic_image_atomic_comp_swap:
result = visit_image_atomic(ctx, instr);
break;
case nir_intrinsic_image_size:
result = visit_image_size(ctx, instr);
break;
case nir_intrinsic_discard:
ctx->shader_info->fs.can_discard = true;
- emit_llvm_intrinsic(ctx, "llvm.AMDGPU.kilp",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.AMDGPU.kilp",
LLVMVoidTypeInContext(ctx->context),
NULL, 0, 0);
break;
case nir_intrinsic_discard_if:
emit_discard_if(ctx, instr);
break;
case nir_intrinsic_memory_barrier:
emit_waitcnt(ctx);
break;
case nir_intrinsic_barrier:
@@ -3389,21 +3351,21 @@ static void visit_tex(struct nir_to_llvm_context *ctx,
nir_tex_instr *instr)
/* Pack texture coordinates */
if (coord) {
address[count++] = coords[0];
if (instr->coord_components > 1)
address[count++] = coords[1];
if (instr->coord_components > 2) {
/* This seems like a bit of a hack - but it passes
Vulkan CTS with it */
if (instr->sampler_dim != GLSL_SAMPLER_DIM_3D &&
instr->op != nir_texop_txf) {
coords[2] = to_float(ctx, coords[2]);
- coords[2] = emit_llvm_intrinsic(ctx,
"llvm.rint.f32", ctx->f32, &coords[2],
+ coords[2] = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.rint.f32", ctx->f32, &coords[2],
1, 0);
coords[2] = to_integer(ctx, coords[2]);
}
address[count++] = coords[2];
}
}
/* Pack LOD */
if ((instr->op == nir_texop_txl || instr->op == nir_texop_txf) && lod) {
address[count++] = lod;
@@ -3771,21 +3733,21 @@ handle_vs_input_decl(struct nir_to_llvm_context *ctx,
buffer_index = LLVMBuildAdd(ctx->builder, ctx->vertex_id,
ctx->base_vertex, "");
for (unsigned i = 0; i < attrib_count; ++i, ++idx) {
t_offset = LLVMConstInt(ctx->i32, index + i, false);
t_list = build_indexed_load_const(ctx, t_list_ptr, t_offset);
args[0] = t_list;
args[1] = LLVMConstInt(ctx->i32, 0, false);
args[2] = buffer_index;
- input = emit_llvm_intrinsic(ctx,
+ input = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.vs.load.input", ctx->v4f32, args, 3,
AC_FUNC_ATTR_READNONE | AC_FUNC_ATTR_NOUNWIND);
for (unsigned chan = 0; chan < 4; chan++) {
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan,
false);
ctx->inputs[radeon_llvm_reg_index_soa(idx, chan)] =
to_integer(ctx,
LLVMBuildExtractElement(ctx->builder,
input, llvm_chan, ""));
}
}
@@ -3817,21 +3779,21 @@ static void interp_fs_input(struct nir_to_llvm_context
*ctx,
intr_name = interp_param ? "llvm.SI.fs.interp" : "llvm.SI.fs.constant";
for (chan = 0; chan < 4; chan++) {
LLVMValueRef args[4];
LLVMValueRef llvm_chan = LLVMConstInt(ctx->i32, chan, false);
args[0] = llvm_chan;
args[1] = attr_number;
args[2] = prim_mask;
args[3] = interp_param;
- result[chan] = emit_llvm_intrinsic(ctx, intr_name,
+ result[chan] = ac_emit_llvm_intrinsic(&ctx->ac, intr_name,
ctx->f32, args, args[3] ? 4
: 3,
AC_FUNC_ATTR_READNONE |
AC_FUNC_ATTR_NOUNWIND);
}
}
static void
handle_fs_input_decl(struct nir_to_llvm_context *ctx,
struct nir_variable *variable)
{
int idx = variable->data.location;
@@ -4085,21 +4047,21 @@ si_llvm_init_export_args(struct nir_to_llvm_context
*ctx,
case V_028714_SPI_SHADER_FP16_ABGR:
args[4] = ctx->i32one;
for (unsigned chan = 0; chan < 2; chan++) {
LLVMValueRef pack_args[2] = {
values[2 * chan],
values[2 * chan + 1]
};
LLVMValueRef packed;
- packed = emit_llvm_intrinsic(ctx,
"llvm.SI.packf16",
+ packed = ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.packf16",
ctx->i32,
pack_args, 2,
AC_FUNC_ATTR_READNONE);
args[chan + 5] = packed;
}
break;
case V_028714_SPI_SHADER_UNORM16_ABGR:
for (unsigned chan = 0; chan < 4; chan++) {
val[chan] = emit_float_saturate(ctx,
values[chan], 0, 1);
val[chan] = LLVMBuildFMul(ctx->builder,
val[chan],
@@ -4253,21 +4215,21 @@ handle_vs_outputs_post(struct nir_to_llvm_context *ctx)
param_count++;
}
si_llvm_init_export_args(ctx, values, target, args);
if (target >= V_008DFC_SQ_EXP_POS &&
target <= (V_008DFC_SQ_EXP_POS + 3)) {
memcpy(pos_args[target - V_008DFC_SQ_EXP_POS],
args, sizeof(args));
} else {
- emit_llvm_intrinsic(ctx,
+ ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
LLVMVoidTypeInContext(ctx->context),
args, 9, 0);
}
}
/* We need to add the position output manually if it's missing. */
if (!pos_args[0][0]) {
pos_args[0][0] = LLVMConstInt(ctx->i32, 0xf, false);
pos_args[0][1] = ctx->i32zero; /* EXEC mask */
@@ -4301,21 +4263,21 @@ handle_vs_outputs_post(struct nir_to_llvm_context *ctx)
pos_idx = 0;
for (i = 0; i < 4; i++) {
if (!pos_args[i][0])
continue;
/* Specify the target we are exporting */
pos_args[i][3] = LLVMConstInt(ctx->i32, V_008DFC_SQ_EXP_POS +
pos_idx++, false);
if (pos_idx == num_pos_exports)
pos_args[i][2] = ctx->i32one;
- emit_llvm_intrinsic(ctx,
+ ac_emit_llvm_intrinsic(&ctx->ac,
"llvm.SI.export",
LLVMVoidTypeInContext(ctx->context),
pos_args[i], 9, 0);
}
ctx->shader_info->vs.pos_exports = num_pos_exports;
ctx->shader_info->vs.param_exports = param_count;
}
static void
@@ -4326,21 +4288,21 @@ si_export_mrt_color(struct nir_to_llvm_context *ctx,
/* Export */
si_llvm_init_export_args(ctx, color, param,
args);
if (is_last) {
args[1] = ctx->i32one; /* whether the EXEC mask is valid */
args[2] = ctx->i32one; /* DONE bit */
} else if (args[0] == ctx->i32zero)
return; /* unnecessary NULL export */
- emit_llvm_intrinsic(ctx, "llvm.SI.export",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
ctx->voidt, args, 9, 0);
}
static void
si_export_mrt_z(struct nir_to_llvm_context *ctx,
LLVMValueRef depth, LLVMValueRef stencil,
LLVMValueRef samplemask)
{
LLVMValueRef args[9];
unsigned mask = 0;
@@ -4370,21 +4332,21 @@ si_export_mrt_z(struct nir_to_llvm_context *ctx,
mask |= 0x04;
}
/* SI (except OLAND) has a bug that it only looks
* at the X writemask component. */
if (ctx->options->chip_class == SI &&
ctx->options->family != CHIP_OLAND)
mask |= 0x01;
args[0] = LLVMConstInt(ctx->i32, mask, false);
- emit_llvm_intrinsic(ctx, "llvm.SI.export",
+ ac_emit_llvm_intrinsic(&ctx->ac, "llvm.SI.export",
ctx->voidt, args, 9, 0);
}
static void
handle_fs_outputs_post(struct nir_to_llvm_context *ctx)
{
unsigned index = 0;
LLVMValueRef depth = NULL, stencil = NULL, samplemask = NULL;
for (unsigned i = 0; i < RADEON_LLVM_MAX_OUTPUTS; ++i) {
--
2.7.4
_______________________________________________
mesa-dev mailing list
mesa-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/mesa-dev
