Hello people, This patch add support for atomics operations in eBPF target using the gcc built-in functions:
__atomic_<operation>_fetch __atomic_fetch_<operation> Please if you have comments, don't hesitate to let me know. Kinds Regards, Guillermo eBPF add support for basic atomic operations, the following gcc built-in functions are implemented for bpf target using both: 32 and 64 bits data types: __atomic_fetch_add __atomic_fetch_sub __atomic_fetch_and __atomic_fetch_xor __atomic_fetch_or __atomic_exchange __atomic_compare_exchange_n Also calls to __atomic_<operation>_fetch are fully supported. New define instructions were added to bpf.md along with its tests for gcc atomic built-in functions. Those operations are fully compliant with: https://gcc.gnu.org/onlinedocs/gcc/_005f_005fatomic-Builtins.html https://www.kernel.org/doc/Documentation/networking/filter.rst This support depends of two previous submissions in CGEN and binutils-gdb projects: https://sourceware.org/pipermail/cgen/2021q3/002774.html https://sourceware.org/pipermail/binutils/2021-August/117798.html gcc/ * config/bpf/bpf.md: Add defines for atomic instructions. gcc/testsuite/ * gcc.target/bpf/atomic-compare-exchange.c: New test. * gcc.target/bpf/atomic-exchange.c: Likewise. * gcc.target/bpf/atomic-fetch-add.c: Likewise. * gcc.target/bpf/atomic-fetch-and.c: Likewise. * gcc.target/bpf/atomic-fetch-or.c: Likewise. * gcc.target/bpf/atomic-fetch-sub.c: Likewise. * gcc.target/bpf/atomic-fetch-xor.c: Likewise. --- gcc/config/bpf/bpf.h | 8 +- gcc/config/bpf/bpf.md | 122 ++++++++++++++++-- gcc/config/bpf/constraints.md | 3 + .../gcc.target/bpf/atomic-compare-exchange.c | 28 ++++ .../gcc.target/bpf/atomic-exchange.c | 19 +++ .../gcc.target/bpf/atomic-fetch-add.c | 26 ++++ .../gcc.target/bpf/atomic-fetch-and.c | 25 ++++ .../gcc.target/bpf/atomic-fetch-or.c | 25 ++++ .../gcc.target/bpf/atomic-fetch-sub.c | 25 ++++ .../gcc.target/bpf/atomic-fetch-xor.c | 25 ++++ 10 files changed, 292 insertions(+), 14 deletions(-) create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-exchange.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c create mode 100644 gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c diff --git a/gcc/config/bpf/bpf.h b/gcc/config/bpf/bpf.h index 82be0c3e190..f8d0f7fb7dd 100644 --- a/gcc/config/bpf/bpf.h +++ b/gcc/config/bpf/bpf.h @@ -176,6 +176,7 @@ enum reg_class { NO_REGS, /* no registers in set. */ + R0, /* register r0. */ ALL_REGS, /* all registers. */ LIM_REG_CLASSES /* max value + 1. */ }; @@ -189,6 +190,7 @@ enum reg_class #define REG_CLASS_NAMES \ { \ "NO_REGS", \ + "R0", \ "ALL_REGS" \ } @@ -202,14 +204,16 @@ enum reg_class #define REG_CLASS_CONTENTS \ { \ 0x00000000, /* NO_REGS */ \ - 0x00000fff, /* ALL_REGS */ \ + 0x00000001, /* R0 */ \ + 0x00000fff, /* ALL_REGS */ \ } /* A C expression whose value is a register class containing hard register REGNO. In general there is more that one such class; choose a class which is "minimal", meaning that no smaller class also contains the register. */ -#define REGNO_REG_CLASS(REGNO) GENERAL_REGS +#define REGNO_REG_CLASS(REGNO) \ + ((REGNO) == 0 ? R0 : GENERAL_REGS) /* A macro whose definition is the name of the class to which a valid base register must belong. A base register is one used in diff --git a/gcc/config/bpf/bpf.md b/gcc/config/bpf/bpf.md index 03830cc250e..f5a4d2d02b4 100644 --- a/gcc/config/bpf/bpf.md +++ b/gcc/config/bpf/bpf.md @@ -25,6 +25,11 @@ (define_c_enum "unspec" [ UNSPEC_LDINDABS UNSPEC_XADD + UNSPEC_XAND + UNSPEC_XOR + UNSPEC_XXOR + UNSPEC_XCHG + UNSPEC_CMPXCHG ]) ;;;; Constants @@ -56,11 +61,10 @@ ;; st generic store instructions for immediates. ;; stx generic store instructions. ;; jmp jump instructions. -;; xadd atomic exchange-and-add instructions. ;; multi multiword sequence (or user asm statements). (define_attr "type" - "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,xadd,multi" + "unknown,alu,alu32,end,ld,lddw,ldx,st,stx,jmp,multi" (const_string "unknown")) ;; Length of instruction in bytes. @@ -506,17 +510,111 @@ "ldabs<ldop>\t%0" [(set_attr "type" "ld")]) -;;;; Atomic increments +;;;; Atomic operations (define_mode_iterator AMO [SI DI]) -(define_insn "atomic_add<AMO:mode>" - [(set (match_operand:AMO 0 "memory_operand" "+m") - (unspec_volatile:AMO - [(plus:AMO (match_dup 0) - (match_operand:AMO 1 "register_operand" "r")) - (match_operand:SI 2 "const_int_operand")] ;; Memory model. - UNSPEC_XADD))] +(define_insn "atomic_fetch_add<AMO:mode>" + [(set (match_operand:AMO 0 "register_operand" "=r") + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") + (match_operand:AMO 2 "nonmemory_operand" "0") + (match_operand:AMO 3 "const_int_operand")] ;; Memory model + UNSPEC_XADD))] + "" + "xadd<mop>\t%1,%0") + +(define_insn "atomic_fetch_and<AMO:mode>" + [(set (match_operand:AMO 0 "register_operand" "=r") + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") + (match_operand:AMO 2 "nonmemory_operand" "0") + (match_operand:AMO 3 "const_int_operand")] + UNSPEC_XAND))] + "" + "xand<mop>\t%1,%0") + +(define_insn "atomic_fetch_or<AMO:mode>" + [(set (match_operand:AMO 0 "register_operand" "=r") + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") + (match_operand:AMO 2 "nonmemory_operand" "0") + (match_operand:AMO 3 "const_int_operand")] + UNSPEC_XOR))] + "" + "xor<mop>\t%1,%0") + +(define_insn "atomic_fetch_xor<AMO:mode>" + [(set (match_operand:AMO 0 "register_operand" "=r") + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") + (match_operand:AMO 2 "nonmemory_operand" "0") + (match_operand:AMO 3 "const_int_operand")] + UNSPEC_XXOR))] + "" + "xxor<mop>\t%1,%0") + +(define_insn "atomic_exchange<AMO:mode>" + [(set (match_operand:AMO 0 "register_operand" "=r") + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") + (match_operand:AMO 2 "nonmemory_operand" "0") + (match_operand:AMO 3 "const_int_operand")] + UNSPEC_XCHG))] + "" + "xchg<mop>\t%1,%0") + +;; eBPF compare and exchange operation atomically compares the +;; value addressed by memory operand(%0) with _R0_(%1), so +;; there is a constraint to load the expected value in mentioned +;; register. If they match it is replaced with desired value(%3). +;; In either case, the value that was there before in %0 is +;; zero-extended and loaded back to R0. + +(define_expand "atomic_compare_and_swap<AMO:mode>" + [(match_operand:SI 0 "register_operand") ;; bool success + (match_operand:AMO 1 "register_operand") ;; old value + (match_operand:AMO 2 "memory_operand") ;; memory + (match_operand:AMO 3 "register_operand") ;; expected + (match_operand:AMO 4 "register_operand") ;; desired + (match_operand:SI 5 "const_int_operand") ;; is_weak (unused) + (match_operand:SI 6 "const_int_operand") ;; success model (unused) + (match_operand:SI 7 "const_int_operand")] ;; failure model (unused) "" - "xadd<mop>\t%0,%1" - [(set_attr "type" "xadd")]) +{ + emit_insn + (gen_atomic_compare_and_swap<AMO:mode>_1 + (operands[1], operands[2], operands[3], operands[4], operands[6])); + + /* Assume success operation, i.e memory operand + is matched with expected value. + */ + emit_move_insn (operands[0], const1_rtx); + rtx_code_label *success_label = gen_label_rtx (); + + /* At this point eBPF xcmp was executed, now we can ask + * for success exchange, and set suitable value for bool + * operand(%0) + */ + emit_cmp_and_jump_insns (operands[1], operands[3], EQ, 0, + GET_MODE (operands[1]), 1, success_label); + emit_move_insn (operands[0], const0_rtx); + + if (success_label) + { + emit_label (success_label); + LABEL_NUSES (success_label) = 1; + } + DONE; +}) + +(define_insn "atomic_compare_and_swap<AMO:mode>_1" + [(set (match_operand:AMO 0 "register_operand" "=t") ;; must be r0 + (unspec_volatile:AMO + [(match_operand:AMO 1 "memory_operand" "+o") ;; memory + (match_operand:AMO 2 "register_operand" "0") ;; expected + (match_operand:AMO 3 "register_operand" "r") ;; desired + (match_operand:SI 4 "const_int_operand")] ;; success (unused) + UNSPEC_CMPXCHG))] + "" + "xcmp<mop>\t%1,%3") diff --git a/gcc/config/bpf/constraints.md b/gcc/config/bpf/constraints.md index 66b7764d775..9606b6e150e 100644 --- a/gcc/config/bpf/constraints.md +++ b/gcc/config/bpf/constraints.md @@ -29,3 +29,6 @@ (define_constraint "S" "A constant call address." (match_code "const,symbol_ref,label_ref,const_int")) + +(define_register_constraint "t" "R0" + "Register r0") diff --git a/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c new file mode 100644 index 00000000000..d522697ec9e --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-compare-exchange.c @@ -0,0 +1,28 @@ +/* { dg-do compile } */ + +long val; +long ptr; +long expected; +long desired; + +void +foo () +{ + int done; + + done = __atomic_compare_exchange_n (&ptr, &expected, desired, + 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + done = __atomic_compare_exchange_n ((int *)&ptr, (int *)&expected, + (int)desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + + done = __atomic_compare_exchange (&ptr, &expected, &desired, + 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + done = __atomic_compare_exchange ((int *)&ptr, (int *)&expected, + (int *)&desired, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); + + done = __sync_bool_compare_and_swap (&ptr, expected, desired); + done = __sync_bool_compare_and_swap ((int*)&ptr, expected, desired); +} + +/* { dg-final { scan-assembler "xcmpdw\t.*" } } */ +/* { dg-final { scan-assembler "xcmpw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-exchange.c b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c new file mode 100644 index 00000000000..f4e60568f7f --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-exchange.c @@ -0,0 +1,19 @@ +/* { dg-do compile } */ + +long val; +long ptr; + +void +foo () +{ + long prev; + + __atomic_exchange(&ptr, &val, &prev, __ATOMIC_RELAXED); + prev = __atomic_exchange_n(&ptr, val, __ATOMIC_RELAXED); + + __atomic_exchange((int *)&ptr, (int *)&val, (int *)&prev, __ATOMIC_RELAXED); + prev = __atomic_exchange_n((int *)&ptr, (int)val, __ATOMIC_RELAXED); +} + +/* { dg-final { scan-assembler "xchgdw\t.*" } } */ +/* { dg-final { scan-assembler "xchgw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c new file mode 100644 index 00000000000..2547396d2a1 --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-add.c @@ -0,0 +1,26 @@ +/* { dg-do compile } */ + +long delta; +long *val; + +void +foo () +{ + long k; + + k = __atomic_fetch_add (val, delta, __ATOMIC_RELAXED); + k = __atomic_fetch_add ((int*)val, delta, __ATOMIC_RELAXED); + + k = __atomic_add_fetch (val, delta, __ATOMIC_RELAXED); + k = __atomic_add_fetch ((int*)val, delta, __ATOMIC_RELAXED); + + k = __sync_fetch_and_add (val, delta, __ATOMIC_RELAXED); + k = __sync_fetch_and_add ((int*)val, delta, __ATOMIC_RELAXED); + + k = __sync_add_and_fetch (val, delta, __ATOMIC_RELAXED); + k = __sync_add_and_fetch ((int*)val, delta, __ATOMIC_RELAXED); + +} + +/* { dg-final { scan-assembler "xadddw\t.*" } } */ +/* { dg-final { scan-assembler "xaddw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c new file mode 100644 index 00000000000..6cc05a824f6 --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-and.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ + +long mask; +long *val; + +void +foo () +{ + long k; + + k = __atomic_fetch_and (val, mask, __ATOMIC_RELAXED); + k = __atomic_fetch_and ((int*)val, mask, __ATOMIC_RELAXED); + + k = __atomic_and_fetch (val, mask, __ATOMIC_RELAXED); + k = __atomic_and_fetch ((int*)val, mask, __ATOMIC_RELAXED); + + k = __sync_fetch_and_and (val, mask, __ATOMIC_RELAXED); + k = __sync_fetch_and_and ((int*)val, mask, __ATOMIC_RELAXED); + + k = __sync_and_and_fetch (val, mask, __ATOMIC_RELAXED); + k = __sync_and_and_fetch ((int*)val, mask, __ATOMIC_RELAXED); +} + +/* { dg-final { scan-assembler "xanddw\t.*" } } */ +/* { dg-final { scan-assembler "xandw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c new file mode 100644 index 00000000000..af9a999e02a --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-or.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ + +long bits; +long *val; + +void +foo () +{ + long k; + + k = __atomic_fetch_or (val, bits, __ATOMIC_RELAXED); + k = __atomic_fetch_or ((int *)val, bits, __ATOMIC_RELAXED); + + k = __atomic_or_fetch (val, bits, __ATOMIC_RELAXED); + k = __atomic_or_fetch ((int*)val, bits, __ATOMIC_RELAXED); + + k = __sync_fetch_and_or (val, bits, __ATOMIC_RELAXED); + k = __sync_fetch_and_or ((int*)val, bits, __ATOMIC_RELAXED); + + k = __sync_or_and_fetch (val, bits, __ATOMIC_RELAXED); + k = __sync_or_and_fetch ((int*)val, bits, __ATOMIC_RELAXED); +} + +/* { dg-final { scan-assembler "xordw\t.*" } } */ +/* { dg-final { scan-assembler "xorw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c new file mode 100644 index 00000000000..6e6a16caa3c --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-sub.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ + +long delta; +long *val; + +void +foo () +{ + long k; + + k = __atomic_fetch_sub (val, delta, __ATOMIC_RELAXED); + k = __atomic_fetch_sub ((int*)val, delta, __ATOMIC_RELAXED); + + k = __atomic_sub_fetch (val, delta, __ATOMIC_RELAXED); + k = __atomic_sub_fetch ((int*)val, delta, __ATOMIC_RELAXED); + + k = __sync_fetch_and_sub (val, delta, __ATOMIC_RELAXED); + k = __sync_fetch_and_sub ((int*)val, delta, __ATOMIC_RELAXED); + + k = __sync_sub_and_fetch (val, delta, __ATOMIC_RELAXED); + k = __sync_sub_and_fetch ((int*)val, delta, __ATOMIC_RELAXED); +} + +/* { dg-final { scan-assembler "xadddw\t.*" } } */ +/* { dg-final { scan-assembler "xaddw\t.*" } } */ diff --git a/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c new file mode 100644 index 00000000000..433600395a6 --- /dev/null +++ b/gcc/testsuite/gcc.target/bpf/atomic-fetch-xor.c @@ -0,0 +1,25 @@ +/* { dg-do compile } */ + +long bits; +long *val; + +void +foo () +{ + long k; + + k = __atomic_fetch_xor (val, bits, __ATOMIC_RELAXED); + k = __atomic_fetch_xor ((int *)val, bits, __ATOMIC_RELAXED); + + k = __atomic_xor_fetch (val, bits, __ATOMIC_RELAXED); + k = __atomic_xor_fetch ((int*)val, bits, __ATOMIC_RELAXED); + + k = __sync_fetch_and_xor (val, bits, __ATOMIC_RELAXED); + k = __sync_fetch_and_xor ((int*)val, bits, __ATOMIC_RELAXED); + + k = __sync_xor_and_fetch (val, bits, __ATOMIC_RELAXED); + k = __sync_xor_and_fetch ((int*)val, bits, __ATOMIC_RELAXED); +} + +/* { dg-final { scan-assembler "xxordw\t.*" } } */ +/* { dg-final { scan-assembler "xxorw\t.*" } } */ -- 2.30.2
