Forceful static keys are used in early entry code where it is unsafe to
defer the sync_core() IPIs, and flagged as such via their ->type field.
Record that information when creating a text_poke_loc. The
text_poke_loc.old field is written to when first iterating a text_poke()
entry, and as such can be (ab)used to store this information at the start
of text_poke_bp_batch().
Signed-off-by: Valentin Schneider <vschn...@redhat.com>
---
arch/x86/include/asm/text-patching.h | 12 ++++++++++--
arch/x86/kernel/alternative.c | 16 ++++++++++------
arch/x86/kernel/jump_label.c | 7 ++++---
3 files changed, 24 insertions(+), 11 deletions(-)
diff --git a/arch/x86/include/asm/text-patching.h
b/arch/x86/include/asm/text-patching.h
index 6259f1937fe77..e34de36cab61e 100644
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -38,9 +38,17 @@ extern void *text_poke_copy(void *addr, const void *opcode,
size_t len);
extern void *text_poke_copy_locked(void *addr, const void *opcode, size_t len,
bool core_ok);
extern void *text_poke_set(void *addr, int c, size_t len);
extern int poke_int3_handler(struct pt_regs *regs);
-extern void text_poke_bp(void *addr, const void *opcode, size_t len, const
void *emulate);
+extern void __text_poke_bp(void *addr, const void *opcode, size_t len, const
void *emulate, bool force_ipi);
+static inline void text_poke_bp(void *addr, const void *opcode, size_t len,
const void *emulate)
+{
+ __text_poke_bp(addr, opcode, len, emulate, false);
+}
-extern void text_poke_queue(void *addr, const void *opcode, size_t len, const
void *emulate);
+extern void __text_poke_queue(void *addr, const void *opcode, size_t len,
const void *emulate, bool force_ipi);
+static inline void text_poke_queue(void *addr, const void *opcode, size_t len,
const void *emulate)
+{
+ __text_poke_queue(addr, opcode, len, emulate, false);
+}
extern void text_poke_finish(void);
#define INT3_INSN_SIZE 1
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index d17518ca19b8b..954c4c0f7fc58 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -2098,7 +2098,10 @@ struct text_poke_loc {
u8 opcode;
const u8 text[POKE_MAX_OPCODE_SIZE];
/* see text_poke_bp_batch() */
- u8 old;
+ union {
+ u8 old;
+ u8 force_ipi;
+ };
};
struct bp_patching_desc {
@@ -2385,7 +2388,7 @@ static void text_poke_bp_batch(struct text_poke_loc *tp,
unsigned int nr_entries
}
static void text_poke_loc_init(struct text_poke_loc *tp, void *addr,
- const void *opcode, size_t len, const void
*emulate)
+ const void *opcode, size_t len, const void
*emulate, bool force_ipi)
{
struct insn insn;
int ret, i = 0;
@@ -2402,6 +2405,7 @@ static void text_poke_loc_init(struct text_poke_loc *tp,
void *addr,
tp->rel_addr = addr - (void *)_stext;
tp->len = len;
tp->opcode = insn.opcode.bytes[0];
+ tp->force_ipi = force_ipi;
if (is_jcc32(&insn)) {
/*
@@ -2493,14 +2497,14 @@ void text_poke_finish(void)
text_poke_flush(NULL);
}
-void __ref text_poke_queue(void *addr, const void *opcode, size_t len, const
void *emulate)
+void __ref __text_poke_queue(void *addr, const void *opcode, size_t len, const
void *emulate, bool force_ipi)
{
struct text_poke_loc *tp;
text_poke_flush(addr);
tp = &tp_vec[tp_vec_nr++];
- text_poke_loc_init(tp, addr, opcode, len, emulate);
+ text_poke_loc_init(tp, addr, opcode, len, emulate, force_ipi);
}
/**
@@ -2514,10 +2518,10 @@ void __ref text_poke_queue(void *addr, const void
*opcode, size_t len, const voi
* dynamically allocated memory. This function should be used when it is
* not possible to allocate memory.
*/
-void __ref text_poke_bp(void *addr, const void *opcode, size_t len, const void
*emulate)
+void __ref __text_poke_bp(void *addr, const void *opcode, size_t len, const
void *emulate, bool force_ipi)
{
struct text_poke_loc tp;
- text_poke_loc_init(&tp, addr, opcode, len, emulate);
+ text_poke_loc_init(&tp, addr, opcode, len, emulate, force_ipi);
text_poke_bp_batch(&tp, 1);
}
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index f5b8ef02d172c..e03a4f56b30fd 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -101,8 +101,8 @@ __jump_label_transform(struct jump_entry *entry,
text_poke_early((void *)jump_entry_code(entry), jlp.code,
jlp.size);
return;
}
-
- text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL);
+ __text_poke_bp((void *)jump_entry_code(entry), jlp.code, jlp.size, NULL,
+ jump_entry_key(entry)->type & JUMP_TYPE_FORCEFUL);
}
static void __ref jump_label_transform(struct jump_entry *entry,
@@ -135,7 +135,8 @@ bool arch_jump_label_transform_queue(struct jump_entry
*entry,
mutex_lock(&text_mutex);
jlp = __jump_label_patch(entry, type);
- text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size,
NULL);
+ __text_poke_queue((void *)jump_entry_code(entry), jlp.code, jlp.size,
NULL,
+ jump_entry_key(entry)->type & JUMP_TYPE_FORCEFUL);
mutex_unlock(&text_mutex);
return true;
}
--
2.43.0
