Add paravirt_patch_runtime() which uses text_poke_late() to patch
paravirt sites.
Also add paravirt_worker() which does the actual insn generation
generate_paravirt() (which uses runtime_patch() to generate the
appropriate native or paravirt insn sequences) and then calls
text_poke_site() to do the actual poking.
CPU0 CPUx
---- ----
patch_worker() patch_worker()
/* Traversal, insn-gen */ text_poke_sync_finish()
tps.patch_worker()
/* = paravirt_worker() */ /*
* wait until:
/* for each patch-site */ * tps->state == PATCH_DONE
generate_paravirt() */
runtime_patch()
text_poke_site()
poke_sync()
... ...
smp_store_release(&tps->state, PATCH_DONE)
Signed-off-by: Ankur Arora <ankur.a.arora@xxxxxxxxxx>
---
arch/x86/include/asm/text-patching.h | 2 +
arch/x86/kernel/alternative.c | 98 +++++++++++++++++++++++++++-
2 files changed, 99 insertions(+), 1 deletion(-)
diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h
index c4b2814f2f9d..e86709a8287e 100644
--- a/arch/x86/include/asm/text-patching.h
+++ b/arch/x86/include/asm/text-patching.h
@@ -21,6 +21,8 @@ static inline void apply_paravirt(struct paravirt_patch_site *start,
#ifndef CONFIG_PARAVIRT_RUNTIME
#define __parainstructions_runtime NULL
#define __parainstructions_runtime_end NULL
+#else
+int paravirt_runtime_patch(void);
#endif
/*
diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index 452d4081eded..1c5acdc4f349 100644
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -1463,7 +1463,9 @@ static void poke_sync(struct text_poke_state *tps, int state, int offset,
/**
* text_poke_site() - called on the primary to patch a single call site.
*
- * Returns after switching tps->state to PATCH_SYNC_DONE.
+ * Called in thread context with tps->state == PATCH_SYNC_DONE where it
+ * takes tps->state through different PATCH_SYNC_* states, returning
+ * after having switched the tps->state back to PATCH_SYNC_DONE.
*/
static void __maybe_unused text_poke_site(struct text_poke_state *tps,
struct text_poke_loc *tp)
@@ -1598,6 +1600,16 @@ static int __maybe_unused text_poke_late(patch_worker_t worker, void *stage)
return ret;
}
+/*
+ * Check if this address is still in scope of this module's .text section.
+ */
+static bool __maybe_unused stale_address(struct alt_module *am, u8 *p)
+{
+ if (p < am->text || p >= am->text_end)
+ return true;
+ return false;
+}
+
#ifdef CONFIG_PARAVIRT_RUNTIME
struct paravirt_stage_entry {
void *dest; /* pv_op destination */
@@ -1654,4 +1666,88 @@ void text_poke_pv_stage_zero(void)
lockdep_assert_held(&text_mutex);
pv_stage.count = 0;
}
+
+/**
+ * generate_paravirt - fill up the insn sequence for a pv-op.
+ *
+ * @tp - address of struct text_poke_loc
+ * @op - the pv-op entry for this location
+ * @site - patch site (kernel or module text)
+ */
+static void generate_paravirt(struct text_poke_loc *tp,
+ struct paravirt_stage_entry *op,
+ struct paravirt_patch_site *site)
+{
+ unsigned int used;
+
+ BUG_ON(site->len > POKE_MAX_OPCODE_SIZE);
+
+ text_poke_loc_init(tp, site->instr, site->instr, site->len, NULL, true);
+
+ /*
+ * Paravirt patches can patch calls (ex. mmu.tlb_flush),
+ * callee_saves(ex. queued_spin_unlock).
+ *
+ * runtime_patch() calls native_patch(), or paravirt_patch()
+ * based on the destination.
+ */
+ used = runtime_patch(site->type, (void *)tp->text, op->dest,
+ (unsigned long)site->instr, site->len);
+
+ /* No good way to recover. */
+ BUG_ON(used < 0);
+
+ /* Pad the rest with nops */
+ add_nops((void *)tp->text + used, site->len - used);
+}
+
+/**
+ * paravirt_worker - generate the paravirt patching
+ * insns and calls text_poke_site() to do the actual patching.
+ */
+static void paravirt_worker(struct text_poke_state *tps)
+{
+ struct paravirt_patch_site *site;
+ struct paravirt_stage *stage = tps->stage;
+ struct paravirt_stage_entry *op = &stage->ops[0];
+ struct alt_module *am;
+ struct text_poke_loc tp;
+ int i;
+
+ list_for_each_entry(am, tps->head, next) {
+ for (site = am->para; site < am->para_end; site++) {
+ if (stale_address(am, site->instr))
+ continue;
+
+ for (i = 0; i < stage->count; i++) {
+ if (op[i].type != site->type)
+ continue;
+
+ generate_paravirt(&tp, &op[i], site);
+
+ text_poke_site(tps, &tp);
+ }
+ }
+ }
+}
+
+/**
+ * paravirt_runtime_patch() -- patch pv-ops, including paired ops.
+ *
+ * Called holding the text_mutex.
+ *
+ * Modify possibly multiple mutually-dependent pv-op callsites
+ * (ex. pv_lock_ops) using stop_machine().
+ *
+ * Return: 0 on success, -errno on failure.
+ */
+int paravirt_runtime_patch(void)
+{
+ lockdep_assert_held(&text_mutex);
+
+ if (!pv_stage.count)
+ return -EINVAL;
+
+ return text_poke_late(paravirt_worker, &pv_stage);
+}
#endif /* CONFIG_PARAVIRT_RUNTIME */
--
2.20.1
