> Implement JIT inlining of the 64-bit bitops kfuncs on arm64.
>
> bpf_clz64(), bpf_ffs64(), bpf_fls64(), and bpf_bitrev64() are always
> inlined using mandatory ARMv8 CLZ/RBIT instructions. bpf_ctz64() is
> inlined via RBIT + CLZ, or via the native CTZ instruction when
> FEAT_CSSC is available. bpf_rol64() and bpf_ror64() are always inlined
> via RORV.
>
> bpf_popcnt64() is not inlined as the native population count instruction
> requires NEON/SIMD registers, which should not be touched from BPF
> programs. It therefore falls back to a regular function call.
>
> Signed-off-by: Leon Hwang <[email protected]>
> ---
> arch/arm64/net/bpf_jit_comp.c | 123 ++++++++++++++++++++++++++++++++++
> 1 file changed, 123 insertions(+)
>
> diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
> index 7a530ea4f5ae..f03f732063d9 100644
> --- a/arch/arm64/net/bpf_jit_comp.c
> +++ b/arch/arm64/net/bpf_jit_comp.c
> @@ -1192,6 +1192,127 @@ static int add_exception_handler(const struct
> bpf_insn *insn,
> return 0;
> }
>
> +static inline u32 a64_clz64(u8 rd, u8 rn)
> +{
> + /*
> + * Arm Architecture Reference Manual for A-profile architecture
> + * (Document number: ARM DDI 0487)
> + *
> + * A64 Base Instruction Descriptions
> + * C6.2 Alphabetical list of A64 base instructions
> + *
> + * C6.2.91 CLZ
> + *
> + * Count leading zeros
> + *
> + * This instruction counts the number of consecutive binary zero
> bits,
> + * starting from the most significant bit in the source register,
> + * and places the count in the destination register.
> + */
> + /* CLZ Xd, Xn */
> + return 0xdac01000 | (rn << 5) | rd;
> +}
> +
> +static inline u32 a64_ctz64(u8 rd, u8 rn)
> +{
> + /*
> + * Arm Architecture Reference Manual for A-profile architecture
> + * (Document number: ARM DDI 0487)
> + *
> + * A64 Base Instruction Descriptions
> + * C6.2 Alphabetical list of A64 base instructions
> + *
> + * C6.2.144 CTZ
> + *
> + * Count trailing zeros
> + *
> + * This instruction counts the number of consecutive binary zero
> bits,
> + * starting from the least significant bit in the source register,
> + * and places the count in the destination register.
> + *
> + * This instruction requires FEAT_CSSC.
> + */
> + /* CTZ Xd, Xn */
> + return 0xdac01800 | (rn << 5) | rd;
> +}
> +
> +static inline u32 a64_rbit64(u8 rd, u8 rn)
> +{
> + /*
> + * Arm Architecture Reference Manual for A-profile architecture
> + * (Document number: ARM DDI 0487)
> + *
> + * A64 Base Instruction Descriptions
> + * C6.2 Alphabetical list of A64 base instructions
> + *
> + * C6.2.320 RBIT
> + *
> + * Reverse bits
> + *
> + * This instruction reverses the bit order in a register.
> + */
> + /* RBIT Xd, Xn */
> + return 0xdac00000 | (rn << 5) | rd;
> +}
Instead of hardcoding the instructions with the above functions, do it the
proper way something like the following patch (not compile tested):
-- >8 --
diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 18c7811774d3..b2696af0b817 100644
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -221,6 +221,9 @@ enum aarch64_insn_data1_type {
AARCH64_INSN_DATA1_REVERSE_16,
AARCH64_INSN_DATA1_REVERSE_32,
AARCH64_INSN_DATA1_REVERSE_64,
+ AARCH64_INSN_DATA1_RBIT,
+ AARCH64_INSN_DATA1_CLZ,
+ AARCH64_INSN_DATA1_CTZ,
};
enum aarch64_insn_data2_type {
@@ -389,6 +392,9 @@ __AARCH64_INSN_FUNCS(rorv, 0x7FE0FC00, 0x1AC02C00)
__AARCH64_INSN_FUNCS(rev16, 0x7FFFFC00, 0x5AC00400)
__AARCH64_INSN_FUNCS(rev32, 0x7FFFFC00, 0x5AC00800)
__AARCH64_INSN_FUNCS(rev64, 0x7FFFFC00, 0x5AC00C00)
+__AARCH64_INSN_FUNCS(rbit, 0x7FFFFC00, 0x5AC00000)
+__AARCH64_INSN_FUNCS(clz, 0x7FFFFC00, 0x5AC01000)
+__AARCH64_INSN_FUNCS(ctz, 0x7FFFFC00, 0x5AC01800)
__AARCH64_INSN_FUNCS(and, 0x7F200000, 0x0A000000)
__AARCH64_INSN_FUNCS(bic, 0x7F200000, 0x0A200000)
__AARCH64_INSN_FUNCS(orr, 0x7F200000, 0x2A000000)
diff --git a/arch/arm64/lib/insn.c b/arch/arm64/lib/insn.c
index 4e298baddc2e..2229ab596cda 100644
--- a/arch/arm64/lib/insn.c
+++ b/arch/arm64/lib/insn.c
@@ -1008,6 +1008,15 @@ u32 aarch64_insn_gen_data1(enum aarch64_insn_register
dst,
}
insn = aarch64_insn_get_rev64_value();
break;
+ case AARCH64_INSN_DATA1_CLZ:
+ insn = aarch64_insn_get_clz_value();
+ break;
+ case AARCH64_INSN_DATA1_RBIT:
+ insn = aarch64_insn_get_rbit_value();
+ break;
+ case AARCH64_INSN_DATA1_CTZ:
+ insn = aarch64_insn_get_ctz_value();
+ break;
default:
pr_err("%s: unknown data1 encoding %d\n", __func__, type);
return AARCH64_BREAK_FAULT;
diff --git a/arch/arm64/net/bpf_jit.h b/arch/arm64/net/bpf_jit.h
index bbea4f36f9f2..af806c39dadb 100644
--- a/arch/arm64/net/bpf_jit.h
+++ b/arch/arm64/net/bpf_jit.h
@@ -248,6 +248,12 @@
#define A64_REV16(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_16)
#define A64_REV32(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, REVERSE_32)
#define A64_REV64(Rd, Rn) A64_DATA1(1, Rd, Rn, REVERSE_64)
+/* Rd = RBIT(Rn) */
+#define A64_RBIT(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, RBIT)
+/* Rd = CLZ(Rn) */
+#define A64_CLZ(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, CLZ)
+/* Rd = CTZ(Rn) */
+#define A64_CTZ(sf, Rd, Rn) A64_DATA1(sf, Rd, Rn, CTZ)
/* Data-processing (2 source) */
/* Rd = Rn OP Rm */
-- 8< --
Thanks,
Puranjay
