[PATCH v2] x86: Support x32 and IBT in heap trampoline

[H.J. Lu]

Add x32 and IBT support to x86 heap trampoline implementation with a
testcase.

2024-02-13  Jakub Jelinek  <ja...@redhat.com>
            H.J. Lu  <hjl.to...@gmail.com>

libgcc/

        PR target/113855
        * config/i386/heap-trampoline.c (trampoline_insns): Add IBT
        support and pad to the multiple of 4 bytes.  Use movabsq
        instead of movabs in comments.  Add -mx32 variant.

gcc/testsuite/

        PR target/113855
        * gcc.dg/heap-trampoline-1.c: New test.
        * lib/target-supports.exp (check_effective_target_heap_trampoline):
        New.
---
 gcc/testsuite/gcc.dg/heap-trampoline-1.c | 23 +++++++++++++
 gcc/testsuite/lib/target-supports.exp    | 12 +++++++
 libgcc/config/i386/heap-trampoline.c     | 42 ++++++++++++++++++++++--
 3 files changed, 74 insertions(+), 3 deletions(-)
 create mode 100644 gcc/testsuite/gcc.dg/heap-trampoline-1.c

diff --git a/gcc/testsuite/gcc.dg/heap-trampoline-1.c 
b/gcc/testsuite/gcc.dg/heap-trampoline-1.c
new file mode 100644
index 00000000000..1aebe00d731
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/heap-trampoline-1.c
@@ -0,0 +1,23 @@
+/* { dg-do run { target heap_trampoline } } */
+/* { dg-options "-ftrampoline-impl=heap" } */
+
+__attribute__((noipa)) int
+bar (int (*fn) (int))
+{
+  return fn (42) + 1;
+}
+
+int
+main ()
+{
+  int a = 0;
+  int foo (int x) { if (x != 42) __builtin_abort (); return ++a; }
+  if (bar (foo) != 2 || a != 1)
+    __builtin_abort ();
+  if (bar (foo) != 3 || a != 2)
+    __builtin_abort ();
+  a = 42;
+  if (bar (foo) != 44 || a != 43)
+    __builtin_abort ();
+  return 0;
+}
diff --git a/gcc/testsuite/lib/target-supports.exp 
b/gcc/testsuite/lib/target-supports.exp
index 6ce8557c9a9..81715999f87 100644
--- a/gcc/testsuite/lib/target-supports.exp
+++ b/gcc/testsuite/lib/target-supports.exp
@@ -13477,3 +13477,15 @@ proc dg-require-python-h { args } {
     eval lappend extra-tool-flags $python_flags
     verbose "After appending, extra-tool-flags: ${extra-tool-flags}" 3
 }
+
+# Return 1 if the target supports heap-trampoline, 0 otherwise.
+proc check_effective_target_heap_trampoline {} {
+    if { [istarget aarch64*-*-linux*]
+        || [istarget i?86-*-darwin*]
+        || [istarget x86_64-*-darwin*]
+        || [istarget i?86-*-linux*]
+        || [istarget x86_64-*-linux*] } {
+       return 1
+    }
+    return 0
+}
diff --git a/libgcc/config/i386/heap-trampoline.c 
b/libgcc/config/i386/heap-trampoline.c
index 1df0aa06108..a8637dc92d3 100644
--- a/libgcc/config/i386/heap-trampoline.c
+++ b/libgcc/config/i386/heap-trampoline.c
@@ -30,28 +30,64 @@ void __gcc_nested_func_ptr_created (void *chain, void 
*func, void *dst);
 void __gcc_nested_func_ptr_deleted (void);
 
 #if __x86_64__
+
+#ifdef __LP64__
 static const uint8_t trampoline_insns[] = {
-  /* movabs $<func>,%r11  */
+#if defined __CET__ && (__CET__ & 1) != 0
+  /* endbr64.  */
+  0xf3, 0x0f, 0x1e, 0xfa,
+#endif
+
+  /* movabsq $<func>,%r11  */
   0x49, 0xbb,
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 
-  /* movabs $<chain>,%r10  */
+  /* movabsq $<chain>,%r10  */
   0x49, 0xba,
   0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 
   /* rex.WB jmpq *%r11  */
-  0x41, 0xff, 0xe3
+  0x41, 0xff, 0xe3,
+
+  /* Pad to the multiple of 4 bytes.  */
+  0x90
 };
+#else
+static const uint8_t trampoline_insns[] = {
+#if defined __CET__ && (__CET__ & 1) != 0
+  /* endbr64.  */
+  0xf3, 0x0f, 0x1e, 0xfa,
+#endif
+
+  /* movl $<func>,%r11d  */
+  0x41, 0xbb,
+  0x00, 0x00, 0x00, 0x00,
+
+  /* movl $<chain>,%r10d  */
+  0x41, 0xba,
+  0x00, 0x00, 0x00, 0x00,
+
+  /* rex.WB jmpq *%r11  */
+  0x41, 0xff, 0xe3,
+
+  /* Pad to the multiple of 4 bytes.  */
+  0x90
+};
+#endif
 
 union ix86_trampoline {
   uint8_t insns[sizeof(trampoline_insns)];
 
   struct __attribute__((packed)) fields {
+#if defined __CET__ && (__CET__ & 1) != 0
+    uint8_t endbr64[4];
+#endif
     uint8_t insn_0[2];
     void *func_ptr;
     uint8_t insn_1[2];
     void *chain_ptr;
     uint8_t insn_2[3];
+    uint8_t pad;
   } fields;
 };
 
-- 
2.43.0