From: Steven Rostedt <srost...@redhat.com>
Add the infrastructure to allow architectures to modify the jump label
locations at compile time. This is mainly for x86, where the jmps may
be either 2 bytes or 5 bytes. Instead of wasting 5 bytes for all jump labels,
this code will let x86 put in a jmp instead of a 5 byte nop. Then the
assembler will make either a 2 byte or 5 byte jmp depending on where
the target is.
At compile time, this code will replace the jmps with either a 2 byte
or 5 byte nop depending on the size of the jmp that was added.
Cc: Jason Baron <jba...@akamai.com>
Signed-off-by: Steven Rostedt <rost...@goodmis.org>
---
Makefile | 7 +
arch/Kconfig | 6 +
scripts/Makefile | 1 +
scripts/Makefile.build | 15 +-
scripts/update_jump_label.c | 341 +++++++++++++++++++++++++++++++++++++++++++
scripts/update_jump_label.h | 208 ++++++++++++++++++++++++++
6 files changed, 576 insertions(+), 2 deletions(-)
create mode 100644 scripts/update_jump_label.c
create mode 100644 scripts/update_jump_label.h
diff --git a/Makefile b/Makefile
index 9262ba8..8ff1c91 100644
--- a/Makefile
+++ b/Makefile
@@ -638,6 +638,13 @@ ifdef CONFIG_DYNAMIC_FTRACE
endif
endif
+ifdef CONFIG_JUMP_LABEL
+ ifdef CONFIG_HAVE_BUILD_TIME_JUMP_LABEL
+ BUILD_UPDATE_JUMP_LABEL := y
+ export BUILD_UPDATE_JUMP_LABEL
+ endif
+endif
+
# We trigger additional mismatches with less inlining
ifdef CONFIG_DEBUG_SECTION_MISMATCH
KBUILD_CFLAGS += $(call cc-option, -fno-inline-functions-called-once)
diff --git a/arch/Kconfig b/arch/Kconfig
index 8d2ae24..dffcb00 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -283,6 +283,12 @@ config HAVE_PERF_USER_STACK_DUMP
access to the user stack pointer which is not unified across
architectures.
+config HAVE_BUILD_TIME_JUMP_LABEL
+ bool
+ help
+ If an arch uses scripts/update_jump_label to patch in jump nops
+ at build time, then it must enable this option.
+
config HAVE_ARCH_JUMP_LABEL
bool
diff --git a/scripts/Makefile b/scripts/Makefile
index 01e7adb..fe548fe 100644
--- a/scripts/Makefile
+++ b/scripts/Makefile
@@ -20,6 +20,7 @@ hostprogs-$(CONFIG_ASN1) += asn1_compiler
HOSTCFLAGS_sortextable.o = -I$(srctree)/tools/include
HOSTCFLAGS_asn1_compiler.o = -I$(srctree)/include
+hostprogs-$(BUILD_UPDATE_JUMP_LABEL) += update_jump_label
always := $(hostprogs-y) $(hostprogs-m)
diff --git a/scripts/Makefile.build b/scripts/Makefile.build
index d5d859c..8084729 100644
--- a/scripts/Makefile.build
+++ b/scripts/Makefile.build
@@ -261,6 +261,15 @@ cmd_modversions =
\
fi;
endif
+ifdef BUILD_UPDATE_JUMP_LABEL
+update_jump_label_source := $(srctree)/scripts/update_jump_label.c \
+ $(srctree)/scripts/update_jump_label.h
+cmd_update_jump_label = \
+ if [ $(@) != "scripts/mod/empty.o" ]; then \
+ $(objtree)/scripts/update_jump_label "$(@)"; \
+ fi;
+endif
+
ifdef CONFIG_FTRACE_MCOUNT_RECORD
ifdef BUILD_C_RECORDMCOUNT
ifeq ("$(origin RECORDMCOUNT_WARN)", "command line")
@@ -297,6 +306,7 @@ define rule_cc_o_c
$(cmd_modversions) \
$(call echo-cmd,record_mcount) \
$(cmd_record_mcount) \
+ $(cmd_update_jump_label) \
scripts/basic/fixdep $(depfile) $@ '$(call make-cmd,cc_o_c)' > \
$(dot-target).tmp; \
rm -f $(depfile); \
@@ -304,13 +314,14 @@ define rule_cc_o_c
endef
# Built-in and composite module parts
-$(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
+$(obj)/%.o: $(src)/%.c $(recordmcount_source) $(update_jump_label_source) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
# Single-part modules are special since we need to mark them in $(MODVERDIR)
-$(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
+$(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) \
+ $(update_jump_label_source) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
@{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod)
diff --git a/scripts/update_jump_label.c b/scripts/update_jump_label.c
new file mode 100644
index 0000000..36d4b43
--- /dev/null
+++ b/scripts/update_jump_label.c
@@ -0,0 +1,341 @@
+/*
+ * update_jump_label.c: replace jmps with nops at compile time.
+ * Copyright 2010 Steven Rostedt <srost...@redhat.com>, Red Hat Inc.
+ * Parsing of the elf file was influenced by recordmcount.c
+ * originally written by and copyright to John F. Reiser
<jrei...@bitwagon.com>.
+ */
+
+/*
+ * Note, this code is originally designed for x86, but may be used by
+ * other archs to do the nop updates at compile time instead of at boot time.
+ * X86 uses this as an optimization, as jmps can be either 2 bytes or 5 bytes.
+ * Inserting a 2 byte where possible helps with both CPU performance and
+ * icache strain.
+ */
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <getopt.h>
+#include <elf.h>
+#include <fcntl.h>
+#include <setjmp.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include <unistd.h>
+
+static int fd_map; /* File descriptor for file being modified. */
+static struct stat sb; /* Remember .st_size, etc. */
+static int mmap_failed; /* Boolean flag. */
+
+static void die(const char *err, const char *fmt, ...)
+{
+ va_list ap;
+
+ if (err)
+ perror(err);
+
+ if (fmt) {
+ va_start(ap, fmt);
+ fprintf(stderr, "Fatal error: ");
+ vfprintf(stderr, fmt, ap);
+ fprintf(stderr, "\n");
+ va_end(ap);
+ }
+
+ exit(1);
+}
+
+static void usage(char **argv)
+{
+ char *arg = argv[0];
+ char *p = arg + strlen(arg);
+
+ while (p >= arg && *p != '/')
+ p--;
+ p++;
+
+ printf("usage: %s file\n"
+ "\n", p);
+ exit(-1);
+}
+
+/* w8rev, w8nat, ...: Handle endianness. */
+
+static uint64_t w8rev(uint64_t const x)
+{
+ return ((0xff & (x >> (0 * 8))) << (7 * 8))
+ | ((0xff & (x >> (1 * 8))) << (6 * 8))
+ | ((0xff & (x >> (2 * 8))) << (5 * 8))
+ | ((0xff & (x >> (3 * 8))) << (4 * 8))
+ | ((0xff & (x >> (4 * 8))) << (3 * 8))
+ | ((0xff & (x >> (5 * 8))) << (2 * 8))
+ | ((0xff & (x >> (6 * 8))) << (1 * 8))
+ | ((0xff & (x >> (7 * 8))) << (0 * 8));
+}
+
+static uint32_t w4rev(uint32_t const x)
+{
+ return ((0xff & (x >> (0 * 8))) << (3 * 8))
+ | ((0xff & (x >> (1 * 8))) << (2 * 8))
+ | ((0xff & (x >> (2 * 8))) << (1 * 8))
+ | ((0xff & (x >> (3 * 8))) << (0 * 8));
+}
+
+static uint32_t w2rev(uint16_t const x)
+{
+ return ((0xff & (x >> (0 * 8))) << (1 * 8))
+ | ((0xff & (x >> (1 * 8))) << (0 * 8));
+}
+
+static uint64_t w8nat(uint64_t const x)
+{
+ return x;
+}
+
+static uint32_t w4nat(uint32_t const x)
+{
+ return x;
+}
+
+static uint32_t w2nat(uint16_t const x)
+{
+ return x;
+}
+
+static uint64_t (*w8)(uint64_t);
+static uint32_t (*w)(uint32_t);
+static uint32_t (*w2)(uint16_t);
+
+/* ulseek, uread, ...: Check return value for errors. */
+
+static off_t
+ulseek(int const fd, off_t const offset, int const whence)
+{
+ off_t const w = lseek(fd, offset, whence);
+ if (w == (off_t)-1)
+ die("lseek", NULL);
+
+ return w;
+}
+
+static size_t
+uread(int const fd, void *const buf, size_t const count)
+{
+ size_t const n = read(fd, buf, count);
+ if (n != count)
+ die("read", NULL);
+
+ return n;
+}
+
+static size_t
+uwrite(int const fd, void const *const buf, size_t const count)
+{
+ size_t const n = write(fd, buf, count);
+ if (n != count)
+ die("write", NULL);
+
+ return n;
+}
+
+static void *
+umalloc(size_t size)
+{
+ void *const addr = malloc(size);
+ if (addr == 0)
+ die("malloc", "malloc failed: %zu bytes\n", size);
+
+ return addr;
+}
+
+/*
+ * Get the whole file as a programming convenience in order to avoid
+ * malloc+lseek+read+free of many pieces. If successful, then mmap
+ * avoids copying unused pieces; else just read the whole file.
+ * Open for both read and write; new info will be appended to the file.
+ * Use MAP_PRIVATE so that a few changes to the in-memory ElfXX_Ehdr
+ * do not propagate to the file until an explicit overwrite at the last.
+ * This preserves most aspects of consistency (all except .st_size)
+ * for simultaneous readers of the file while we are appending to it.
+ * However, multiple writers still are bad. We choose not to use
+ * locking because it is expensive and the use case of kernel build
+ * makes multiple writers unlikely.
+ */
+static void *mmap_file(char const *fname)
+{
+ void *addr;
+
+ fd_map = open(fname, O_RDWR);
+ if (fd_map < 0 || fstat(fd_map, &sb) < 0)
+ die(fname, "failed to open file");
+
+ if (!S_ISREG(sb.st_mode))
+ die(NULL, "not a regular file: %s\n", fname);
+
+ addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE,
+ fd_map, 0);
+
+ mmap_failed = 0;
+ if (addr == MAP_FAILED) {
+ mmap_failed = 1;
+ addr = umalloc(sb.st_size);
+ uread(fd_map, addr, sb.st_size);
+ }
+ return addr;
+}
+
+static void munmap_file(void *addr)
+{
+ if (!mmap_failed)
+ munmap(addr, sb.st_size);
+ else
+ free(addr);
+ close(fd_map);
+}
+
+/* P6_NOP5_ATOMIC */
+static unsigned char ideal_nop5_x86_64[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
+
+/* GENERIC_NOP5_ATOMIC */
+static unsigned char ideal_nop5_x86_32[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
+
+/* P6_NOP2 */
+static unsigned char ideal_nop2_x86[2] = { 0x66, 0x90 };
+
+static unsigned char *ideal_nop;
+
+static int (*make_nop)(void *map, size_t const offset);
+
+static int make_nop_x86(void *map, size_t const offset)
+{
+ unsigned char *op;
+ unsigned char *nop;
+ int size;
+
+ /* Determine which type of jmp this is 2 byte or 5. */
+ op = map + offset;
+ switch (*op) {
+ case 0xeb: /* 2 byte */
+ size = 2;
+ nop = ideal_nop2_x86;
+ break;
+ case 0xe9: /* 5 byte */
+ size = 5;
+ nop = ideal_nop;
+ break;
+ default:
+ die(NULL, "Bad jump label section (bad op %x)\n", *op);
+ __builtin_unreachable();
+ }
+
+ /* convert to nop */
+ ulseek(fd_map, offset, SEEK_SET);
+ uwrite(fd_map, nop, size);
+ return 0;
+}
+
+/* 32 bit and 64 bit are very similar */
+#include "update_jump_label.h"
+#define UPDATE_JUMP_LABEL_64
+#include "update_jump_label.h"
+
+static int do_file(const char *fname)
+{
+ Elf32_Ehdr *const ehdr = mmap_file(fname);
+ unsigned int reltype = 0;
+
+ w = w4nat;
+ w2 = w2nat;
+ w8 = w8nat;
+ switch (ehdr->e_ident[EI_DATA]) {
+ static unsigned int const endian = 1;
+ default:
+ die(NULL, "unrecognized ELF data encoding %d: %s\n",
+ ehdr->e_ident[EI_DATA], fname);
+ break;
+ case ELFDATA2LSB:
+ if (*(unsigned char const *)&endian != 1) {
+ /* main() is big endian, file.o is little endian. */
+ w = w4rev;
+ w2 = w2rev;
+ w8 = w8rev;
+ }
+ break;
+ case ELFDATA2MSB:
+ if (*(unsigned char const *)&endian != 0) {
+ /* main() is little endian, file.o is big endian. */
+ w = w4rev;
+ w2 = w2rev;
+ w8 = w8rev;
+ }
+ break;
+ } /* end switch */
+
+ if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0 ||
+ w2(ehdr->e_type) != ET_REL ||
+ ehdr->e_ident[EI_VERSION] != EV_CURRENT)
+ die(NULL, "unrecognized ET_REL file %s\n", fname);
+
+ switch (w2(ehdr->e_machine)) {
+ default:
+ die(NULL, "unrecognized e_machine %d %s\n",
+ w2(ehdr->e_machine), fname);
+ break;
+ case EM_386:
+ reltype = R_386_32;
+ make_nop = make_nop_x86;
+ ideal_nop = ideal_nop5_x86_32;
+ break;
+ case EM_ARM: reltype = R_ARM_ABS32;
+ break;
+ case EM_IA_64: reltype = R_IA64_IMM64; break;
+ case EM_MIPS: /* reltype: e_class */ break;
+ case EM_PPC: reltype = R_PPC_ADDR32; break;
+ case EM_PPC64: reltype = R_PPC64_ADDR64; break;
+ case EM_S390: /* reltype: e_class */ break;
+ case EM_SH: reltype = R_SH_DIR32; break;
+ case EM_SPARCV9: reltype = R_SPARC_64; break;
+ case EM_X86_64:
+ make_nop = make_nop_x86;
+ ideal_nop = ideal_nop5_x86_64;
+ reltype = R_X86_64_64;
+ break;
+ } /* end switch */
+
+ switch (ehdr->e_ident[EI_CLASS]) {
+ default:
+ die(NULL, "unrecognized ELF class %d %s\n",
+ ehdr->e_ident[EI_CLASS], fname);
+ break;
+ case ELFCLASS32:
+ if (w2(ehdr->e_ehsize) != sizeof(Elf32_Ehdr)
+ || w2(ehdr->e_shentsize) != sizeof(Elf32_Shdr))
+ die(NULL, "unrecognized ET_REL file: %s\n", fname);
+
+ do_func32(ehdr, fname, reltype);
+ break;
+ case ELFCLASS64: {
+ Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr;
+ if (w2(ghdr->e_ehsize) != sizeof(Elf64_Ehdr)
+ || w2(ghdr->e_shentsize) != sizeof(Elf64_Shdr))
+ die(NULL, "unrecognized ET_REL file: %s\n", fname);
+
+ do_func64(ghdr, fname, reltype);
+ break;
+ }
+ } /* end switch */
+
+ munmap_file(ehdr);
+ return 0;
+}
+
+int main(int argc, char **argv)
+{
+ if (argc != 2)
+ usage(argv);
+
+ return do_file(argv[1]);
+}
+
diff --git a/scripts/update_jump_label.h b/scripts/update_jump_label.h
new file mode 100644
index 0000000..181bd5b
--- /dev/null
+++ b/scripts/update_jump_label.h
@@ -0,0 +1,208 @@
+/*
+ * update_jump_label.h
+ *
+ * This code was based off of code from recordmcount.c written by
+ * Copyright 2009 John F. Reiser <jrei...@bitwagon.com>. All rights reserved.
+ *
+ * The original code had the same algorithms for both 32bit
+ * and 64bit ELF files, but the code was duplicated to support
+ * the difference in structures that were used. This
+ * file creates a macro of everything that is different between
+ * the 64 and 32 bit code, such that by including this header
+ * twice we can create both sets of functions by including this
+ * header once with UPDATE_JUMP_LABEL_64 undefined, and again with
+ * it defined.
+ *
+ * Copyright 2010 Steven Rostedt <srost...@redhat.com>, Red Hat Inc.
+ *
+ * Licensed under the GNU General Public License, version 2 (GPLv2).
+ */
+
+#undef EBITS
+#undef _w
+
+#ifdef UPDATE_JUMP_LABEL_64
+# define EBITS 64
+# define _w w8
+#else
+# define EBITS 32
+# define _w w
+#endif
+
+#define _FBITS(x, e) x##e
+#define FBITS(x, e) _FBITS(x, e)
+#define FUNC(x) FBITS(x, EBITS)
+
+#undef Elf_Ehdr
+#undef Elf_Shdr
+#undef Elf_Rel
+#undef Elf_Rela
+#undef Elf_Sym
+#undef ELF_R_SYM
+#undef ELF_R_TYPE
+
+#define __ATTACH(x, y, z) x##y##z
+#define ATTACH(x, y, z) __ATTACH(x, y, z)
+
+#define Elf_Ehdr ATTACH(Elf, EBITS, _Ehdr)
+#define Elf_Shdr ATTACH(Elf, EBITS, _Shdr)
+#define Elf_Rel ATTACH(Elf, EBITS, _Rel)
+#define Elf_Rela ATTACH(Elf, EBITS, _Rela)
+#define Elf_Sym ATTACH(Elf, EBITS, _Sym)
+#define uint_t ATTACH(uint, EBITS, _t)
+#define ELF_R_SYM ATTACH(ELF, EBITS, _R_SYM)
+#define ELF_R_TYPE ATTACH(ELF, EBITS, _R_TYPE)
+
+#undef get_shdr
+#define get_shdr(ehdr) ((Elf_Shdr *)(_w((ehdr)->e_shoff) + (void *)(ehdr)))
+
+#undef get_section_loc
+#define get_section_loc(ehdr, shdr)(_w((shdr)->sh_offset) + (void *)(ehdr))
+
+/* Find relocation section hdr for a given section */
+static const Elf_Shdr *
+FUNC(find_relhdr)(const Elf_Ehdr *ehdr, const Elf_Shdr *shdr)
+{
+ const Elf_Shdr *shdr0 = get_shdr(ehdr);
+ int nhdr = w2(ehdr->e_shnum);
+ const Elf_Shdr *hdr;
+ int i;
+
+ for (hdr = shdr0, i = 0; i < nhdr; hdr = &shdr0[++i]) {
+ if (w(hdr->sh_type) != SHT_REL &&
+ w(hdr->sh_type) != SHT_RELA)
+ continue;
+
+ /*
+ * The relocation section's info field holds
+ * the section index that it represents.
+ */
+ if (shdr == &shdr0[w(hdr->sh_info)])
+ return hdr;
+ }
+ return NULL;
+}
+
+/* Find a section headr based on name and type */
+static const Elf_Shdr *
+FUNC(find_shdr)(const Elf_Ehdr *ehdr, const char *name, uint_t type)
+{
+ const Elf_Shdr *shdr0 = get_shdr(ehdr);
+ const Elf_Shdr *shstr = &shdr0[w2(ehdr->e_shstrndx)];
+ const char *shstrtab = (char *)get_section_loc(ehdr, shstr);
+ int nhdr = w2(ehdr->e_shnum);
+ const Elf_Shdr *hdr;
+ const char *hdrname;
+ int i;
+
+ for (hdr = shdr0, i = 0; i < nhdr; hdr = &shdr0[++i]) {
+ if (w(hdr->sh_type) != type)
+ continue;
+
+ /* If we are just looking for a section by type (ie. SYMTAB) */
+ if (!name)
+ return hdr;
+
+ hdrname = &shstrtab[w(hdr->sh_name)];
+ if (strcmp(hdrname, name) == 0)
+ return hdr;
+ }
+ return NULL;
+}
+
+static void
+FUNC(section_update)(const Elf_Ehdr *ehdr, const Elf_Shdr *symhdr,
+ unsigned shtype, const Elf_Rel *rel, void *data)
+{
+ const Elf_Shdr *shdr0 = get_shdr(ehdr);
+ const Elf_Shdr *targethdr;
+ const Elf_Rela *rela;
+ const Elf_Sym *syment;
+ uint_t offset = _w(rel->r_offset);
+ uint_t info = _w(rel->r_info);
+ uint_t sym = ELF_R_SYM(info);
+ uint_t type = ELF_R_TYPE(info);
+ uint_t addend;
+ uint_t targetloc;
+
+ if (shtype == SHT_RELA) {
+ rela = (const Elf_Rela *)rel;
+ addend = _w(rela->r_addend);
+ } else
+ addend = _w(*(int *)(data + offset));
+
+ syment = (const Elf_Sym *)get_section_loc(ehdr, symhdr);
+ targethdr = &shdr0[w2(syment[sym].st_shndx)];
+ targetloc = _w(targethdr->sh_offset);
+
+ /* TODO, need a separate function for all archs */
+ if (type != R_386_32)
+ die(NULL, "Arch relocation type %d not supported", type);
+
+ targetloc += addend;
+
+ *(uint_t *)(data + offset) = targetloc;
+}
+
+/* Overall supervision for Elf32 ET_REL file. */
+static void
+FUNC(do_func)(Elf_Ehdr *ehdr, char const *const fname, unsigned const reltype)
+{
+ const Elf_Shdr *jlshdr;
+ const Elf_Shdr *jlrhdr;
+ const Elf_Shdr *symhdr;
+ const Elf_Rel *rel;
+ unsigned size;
+ unsigned cnt;
+ unsigned i;
+ uint_t type;
+ void *jdata;
+ void *data;
+
+ jlshdr = FUNC(find_shdr)(ehdr, "__jump_table", SHT_PROGBITS);
+ if (!jlshdr)
+ return;
+
+ jlrhdr = FUNC(find_relhdr)(ehdr, jlshdr);
+ if (!jlrhdr)
+ return;
+
+ /*
+ * Create and fill in the __jump_table section and use it to
+ * find the offsets into the text that we want to update.
+ * We create it so that we do not depend on the order of the
+ * relocations, and use the table directly, as it is broken
+ * up into sections.
+ */
+ size = _w(jlshdr->sh_size);
+ data = umalloc(size);
+
+ jdata = (void *)get_section_loc(ehdr, jlshdr);
+ memcpy(data, jdata, size);
+
+ cnt = _w(jlrhdr->sh_size) / w(jlrhdr->sh_entsize);
+
+ rel = (const Elf_Rel *)get_section_loc(ehdr, jlrhdr);
+
+ /* Is this as Rel or Rela? */
+ type = w(jlrhdr->sh_type);
+
+ symhdr = FUNC(find_shdr)(ehdr, NULL, SHT_SYMTAB);
+
+ for (i = 0; i < cnt; i++) {
+ FUNC(section_update)(ehdr, symhdr, type, rel, data);
+ rel = (void *)rel + w(jlrhdr->sh_entsize);
+ }
+
+ /*
+ * This is specific to x86. The jump_table is stored in three
+ * long words. The first is the location of the jmp target we
+ * must update.
+ */
+ cnt = size / sizeof(uint_t);
+
+ for (i = 0; i < cnt; i += 3)
+ make_nop((void *)ehdr, *(uint_t *)(data + i * sizeof(uint_t)));
+
+ free(data);
+}
--
1.7.10.4
--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
