The arm64 JIT rejected BPF_LD | BPF_ABS and BPF_LD | BPF_IND with "invalid opcode", so cBPF programs converted by rte_bpf_convert() could not be JITed. Add these opcodes, mirroring the x86 JIT: a fast path for data held in the first mbuf segment, and a __rte_pktmbuf_read() slow path for everything else.
emit_call() materializes the helper address with a variable number of instructions, so the block sizes are not known up front. Size the three blocks in a dry run, then emit for real with the branches resolved. For BPF_IND the offset is a runtime value, so it is checked for being negative or too large for the uint32_t off argument of __rte_pktmbuf_read(). For BPF_ABS the offset is the instruction immediate, so those checks are dropped and a negative immediate compiles to a return of zero. Programs using these opcodes use the call register layout, since the slow path makes a function call. Bugzilla ID: 1427 Signed-off-by: Stephen Hemminger <[email protected]> Acked-by: Marat Khalili <[email protected]> Acked-by: Konstantin Ananyev <[email protected]> --- lib/bpf/bpf_jit_arm64.c | 188 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 187 insertions(+), 1 deletion(-) diff --git a/lib/bpf/bpf_jit_arm64.c b/lib/bpf/bpf_jit_arm64.c index 51906c7f0d..fc922006eb 100644 --- a/lib/bpf/bpf_jit_arm64.c +++ b/lib/bpf/bpf_jit_arm64.c @@ -8,6 +8,7 @@ #include <rte_common.h> #include <rte_byteorder.h> +#include <rte_debug.h> #include "bpf_impl.h" @@ -1133,6 +1134,173 @@ emit_branch(struct a64_jit_ctx *ctx, uint8_t op, uint32_t i, int16_t off) emit_b_cond(ctx, ebpf_to_a64_cond(op), jump_offset_get(ctx, i, off)); } +/* LD_ABS/LD_IND code block offsets (in arm64 instructions) */ +enum { + LDMB_FAST_OFS, /* fast path */ + LDMB_SLOW_OFS, /* slow path */ + LDMB_FIN_OFS, /* common tail */ + LDMB_OFS_NUM +}; +/* + * Helper for emit_ld_mbuf(): fast path. + * Compute the packet offset; if it lies inside the first segment leave the + * data pointer in R0, otherwise branch to the slow path. + */ +static void +emit_ldmb_fast_path(struct a64_jit_ctx *ctx, uint8_t src, uint8_t mode, + uint32_t sz, int32_t imm, const uint32_t ofs[LDMB_OFS_NUM]) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); + uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); + uint8_t tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2); + uint8_t tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3); + + emit_mov_imm(ctx, 1, tmp1, imm); + + if (mode == BPF_IND) { + /* off = imm + src */ + emit_add(ctx, 1, tmp1, src); + + /* + * src is a runtime value, so off may be negative. + * That would pass the signed check below and read before the segment, + * so route it to the slow path which rejects it. + */ + emit_cmp(ctx, 1, tmp1, A64_ZR); + emit_b_cond(ctx, A64_MI, (int32_t)(ofs[LDMB_SLOW_OFS] - ctx->idx)); + } + + /* if ((int64_t)(mbuf->data_len - off) < sz) goto slow_path */ + emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_len)); + emit_ldr(ctx, BPF_H, tmp2, r6, tmp2); + emit_sub(ctx, 1, tmp2, tmp1); + emit_mov_imm(ctx, 1, tmp3, sz); + emit_cmp(ctx, 1, tmp2, tmp3); + emit_b_cond(ctx, A64_LT, (int32_t)(ofs[LDMB_SLOW_OFS] - ctx->idx)); + + /* R0 = mbuf->buf_addr + mbuf->data_off + off */ + emit_mov_imm(ctx, 1, tmp2, offsetof(struct rte_mbuf, data_off)); + emit_ldr(ctx, BPF_H, tmp2, r6, tmp2); + emit_mov_imm(ctx, 1, r0, offsetof(struct rte_mbuf, buf_addr)); + emit_ldr(ctx, EBPF_DW, r0, r6, r0); + emit_add(ctx, 1, r0, tmp2); + emit_add(ctx, 1, r0, tmp1); + + emit_b(ctx, (int32_t)(ofs[LDMB_FIN_OFS] - ctx->idx)); +} + +/* + * Helper for emit_ld_mbuf(): common tail. + * Load the value pointed to by R0 and convert from network byte order. + */ +static void +emit_ldmb_fin(struct a64_jit_ctx *ctx, uint8_t opsz, uint32_t sz) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + + emit_ldr(ctx, opsz, r0, r0, A64_ZR); + if (opsz != BPF_B) + emit_be(ctx, r0, sz * 8); +} + +/* + * Helper for emit_ld_mbuf(): slow path. + * R0 = __rte_pktmbuf_read(mbuf, off, sz, buf); return 0 if NULL. + * The scratch buffer is the space reserved by __rte_bpf_validate() at the + * bottom of the eBPF stack frame, i.e. (frame_pointer - stack_ofs). + */ +static void +emit_ldmb_slow_path(struct a64_jit_ctx *ctx, uint8_t mode, uint32_t sz, + uint32_t stack_ofs) +{ + uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0); + uint8_t r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6); + uint8_t fp = ebpf_to_a64_reg(ctx, EBPF_FP); + uint8_t tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); + uint8_t tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2); + + if (mode == BPF_IND) { + /* + * __rte_pktmbuf_read() takes a uint32_t off, so a 64-bit off + * that does not fit would be silently truncated. Return 0 if + * out of range; this also catches the negative off routed here + * by the fast path. + */ + emit_mov_imm(ctx, 1, tmp2, UINT32_MAX); + emit_cmp(ctx, 1, tmp1, tmp2); + emit_b_cond(ctx, A64_LS, 3); /* in range: do the call */ + emit_mov_imm(ctx, 1, r0, 0); + emit_b(ctx, (ctx->program_start + ctx->program_sz) - ctx->idx); + } + + /* arguments of __rte_pktmbuf_read(mbuf, off, len, buf) */ + emit_mov_64(ctx, A64_R(1), tmp1); /* off (held in tmp1) */ + emit_mov_64(ctx, A64_R(0), r6); /* mbuf */ + emit_mov_imm(ctx, 0, A64_R(2), sz); /* len */ + emit_sub_imm_64(ctx, A64_R(3), fp, stack_ofs); /* buf */ + + emit_call(ctx, tmp1, (void *)(uintptr_t)__rte_pktmbuf_read); + emit_return_zero_if_src_zero(ctx, 1, r0); +} + +/* + * Emit code for BPF_LD | BPF_ABS and BPF_LD | BPF_IND packet loads: + * + * off = imm (+ src for BPF_IND) + * if (off >= 0 && mbuf->data_len - off >= sz) -- fast path + * ptr = mbuf->buf_addr + mbuf->data_off + off; + * else -- slow path + * if ((uint64_t)off > UINT32_MAX) + * return 0; + * ptr = __rte_pktmbuf_read(mbuf, off, sz, buf); + * if (ptr == NULL) + * return 0; + * R0 = ntoh(*(size *)ptr); -- common tail + * + * For BPF_ABS the offset is a constant, so the range checks on off are only + * emitted for BPF_IND. + * + * The three blocks are sized in a dry run so the forward branches can be + * resolved, then emitted for real. Both runs see the same immediates, so + * they emit the same instructions; the recorded offsets are only valid + * while that holds, hence the checks. + */ +static void +emit_ld_mbuf(struct a64_jit_ctx *ctx, uint8_t op, uint8_t src, int32_t imm, + uint32_t stack_ofs) +{ + uint8_t mode = BPF_MODE(op); + uint8_t opsz = BPF_SIZE(op); + uint32_t sz = bpf_size(opsz); + uint32_t ofs[LDMB_OFS_NUM]; + + /* a constant negative offset is never inside the packet */ + if (mode == BPF_ABS && imm < 0) { + emit_mov_imm(ctx, 1, ebpf_to_a64_reg(ctx, EBPF_REG_0), 0); + emit_b(ctx, (ctx->program_start + ctx->program_sz) - ctx->idx); + return; + } + + /* seed offsets so the dry-run branches stay in range */ + ofs[LDMB_FAST_OFS] = ofs[LDMB_SLOW_OFS] = ofs[LDMB_FIN_OFS] = ctx->idx; + + /* dry run to record block offsets */ + emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs); + ofs[LDMB_SLOW_OFS] = ctx->idx; + emit_ldmb_slow_path(ctx, mode, sz, stack_ofs); + ofs[LDMB_FIN_OFS] = ctx->idx; + emit_ldmb_fin(ctx, opsz, sz); + + /* rewind and emit for real with resolved offsets */ + ctx->idx = ofs[LDMB_FAST_OFS]; + emit_ldmb_fast_path(ctx, src, mode, sz, imm, ofs); + RTE_VERIFY(ctx->idx == ofs[LDMB_SLOW_OFS]); + emit_ldmb_slow_path(ctx, mode, sz, stack_ofs); + RTE_VERIFY(ctx->idx == ofs[LDMB_FIN_OFS]); + emit_ldmb_fin(ctx, opsz, sz); +} + static void check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) { @@ -1145,8 +1313,17 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) op = ins->code; switch (op) { - /* Call imm */ + /* + * BPF_ABS/BPF_IND can fall through to __rte_pktmbuf_read(), + * so they need the call-clobbered register layout as well. + */ case (BPF_JMP | EBPF_CALL): + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): ctx->foundcall = 1; return; } @@ -1348,6 +1525,15 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) emit_mov_imm(ctx, 1, dst, u64); i++; break; + /* R0 = ntoh(*(size *)(mbuf data + (src) + imm)) */ + case (BPF_LD | BPF_ABS | BPF_B): + case (BPF_LD | BPF_ABS | BPF_H): + case (BPF_LD | BPF_ABS | BPF_W): + case (BPF_LD | BPF_IND | BPF_B): + case (BPF_LD | BPF_IND | BPF_H): + case (BPF_LD | BPF_IND | BPF_W): + emit_ld_mbuf(ctx, op, src, imm, bpf->stack_sz); + break; /* *(size *)(dst + off) = src */ case (BPF_STX | BPF_MEM | BPF_B): case (BPF_STX | BPF_MEM | BPF_H): -- 2.53.0
