Add functions to support the FEAT_TLBIRANGE ARMv8.4 feature that adds
TLB invalidation instructions to invalidate ranges of addresses.
Signed-off-by: Rebecca Cran <rebe...@nuviainc.com>
---
accel/tcg/cputlb.c | 128 +++++++++++++++++++-
include/exec/exec-all.h | 46 +++++++
2 files changed, 171 insertions(+), 3 deletions(-)
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 8a7b779270a4..9381745f2528 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -709,7 +709,7 @@ void tlb_flush_page_all_cpus_synced(CPUState *src,
target_ulong addr)
tlb_flush_page_by_mmuidx_all_cpus_synced(src, addr, ALL_MMUIDX_BITS);
}
-static void tlb_flush_page_bits_locked(CPUArchState *env, int midx,
+static bool tlb_flush_page_bits_locked(CPUArchState *env, int midx,
target_ulong page, unsigned bits)
{
CPUTLBDesc *d = &env_tlb(env)->d[midx];
@@ -729,7 +729,7 @@ static void tlb_flush_page_bits_locked(CPUArchState *env,
int midx,
TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
midx, page, mask);
tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
- return;
+ return true;
}
/* Check if we need to flush due to large pages. */
@@ -738,13 +738,14 @@ static void tlb_flush_page_bits_locked(CPUArchState *env,
int midx,
TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
midx, d->large_page_addr, d->large_page_mask);
tlb_flush_one_mmuidx_locked(env, midx, get_clock_realtime());
- return;
+ return true;
}
if (tlb_flush_entry_mask_locked(tlb_entry(env, midx, page), page, mask)) {
tlb_n_used_entries_dec(env, midx);
}
tlb_flush_vtlb_page_mask_locked(env, midx, page, mask);
+ return false;
}
typedef struct {
@@ -943,6 +944,127 @@ void
tlb_flush_page_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
}
}
+typedef struct {
+ target_ulong addr;
+ target_ulong length;
+ uint16_t idxmap;
+ uint16_t bits;
+} TLBFlushPageRangeBitsByMMUIdxData;
+
+static void
+tlb_flush_page_range_bits_by_mmuidx_async_0(CPUState *cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits)
+{
+ CPUArchState *env = cpu->env_ptr;
+ bool full_flush;
+ int mmu_idx;
+ target_ulong page;
+
+ assert_cpu_is_self(cpu);
+
+ tlb_debug("page addr:" TARGET_FMT_lx "/%u len: " TARGET_FMT_lx
+ " mmu_map:0x%x\n",
+ addr, bits, length, idxmap);
+
+ qemu_spin_lock(&env_tlb(env)->c.lock);
+ for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+ if ((idxmap >> mmu_idx) & 1) {
+ for (page = addr; page < (addr + length); page +=
TARGET_PAGE_SIZE) {
+ full_flush = tlb_flush_page_bits_locked(env, mmu_idx,
+ page, bits);
+ if (full_flush) {
+ break;
+ }
+ }
+ }
+ }
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
+
+ for (page = addr; page < (addr + length); page += TARGET_PAGE_SIZE) {
+ tb_flush_jmp_cache(cpu, page);
+ }
+}
+
+static void
+tlb_flush_page_range_bits_by_mmuidx_async_1(CPUState *cpu,
+ run_on_cpu_data data)
+{
+ TLBFlushPageRangeBitsByMMUIdxData *d = data.host_ptr;
+
+ tlb_flush_page_range_bits_by_mmuidx_async_0(cpu, d->addr, d->length,
+ d->idxmap, d->bits);
+
+ g_free(d);
+}
+
+void tlb_flush_page_range_bits_by_mmuidx(CPUState *cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits)
+{
+ TLBFlushPageRangeBitsByMMUIdxData d;
+ TLBFlushPageRangeBitsByMMUIdxData *p;
+
+ /* This should already be page aligned */
+ addr &= TARGET_PAGE_BITS;
+
+ d.addr = addr & TARGET_PAGE_MASK;
+ d.idxmap = idxmap;
+ d.bits = bits;
+ d.length = length;
+
+ if (qemu_cpu_is_self(cpu)) {
+ tlb_flush_page_range_bits_by_mmuidx_async_0(cpu, addr, length,
+ idxmap, bits);
+ } else {
+ p = g_new(TLBFlushPageRangeBitsByMMUIdxData, 1);
+
+ /* Allocate a structure, freed by the worker. */
+ *p = d;
+ async_run_on_cpu(cpu, tlb_flush_page_range_bits_by_mmuidx_async_1,
+ RUN_ON_CPU_HOST_PTR(p));
+ }
+}
+
+void tlb_flush_page_range_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits)
+{
+ TLBFlushPageRangeBitsByMMUIdxData d;
+ TLBFlushPageRangeBitsByMMUIdxData *p;
+ CPUState *dst_cpu;
+
+ /* This should already be page aligned */
+ addr &= TARGET_PAGE_BITS;
+
+ d.addr = addr;
+ d.idxmap = idxmap;
+ d.bits = bits;
+ d.length = length;
+
+ /* Allocate a separate data block for each destination cpu. */
+ CPU_FOREACH(dst_cpu) {
+ if (dst_cpu != src_cpu) {
+ p = g_new(TLBFlushPageRangeBitsByMMUIdxData, 1);
+ *p = d;
+ async_run_on_cpu(dst_cpu,
+ tlb_flush_page_range_bits_by_mmuidx_async_1,
+ RUN_ON_CPU_HOST_PTR(p));
+ }
+ }
+
+ p = g_new(TLBFlushPageRangeBitsByMMUIdxData, 1);
+ *p = d;
+ async_safe_run_on_cpu(src_cpu, tlb_flush_page_range_bits_by_mmuidx_async_1,
+ RUN_ON_CPU_HOST_PTR(p));
+}
+
/* update the TLBs so that writes to code in the virtual page 'addr'
can be detected */
void tlb_protect_code(ram_addr_t ram_addr)
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 6b036cae8f65..a7ff35efb865 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -212,6 +212,37 @@ void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu,
target_ulong addr,
*/
void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr,
uint16_t idxmap);
+/**
+ * tlb_flush_page_range_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of start of page range to be flushed
+ * @length: the number of bytes to be flushed
+ * @idxmap: bitmap of MMU indexes to flush
+ *
+ * Flush a range of pages from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_page_range_bits_by_mmuidx(CPUState *cpu, target_ulong addr,
+ target_ulong length, uint16_t idxmap,
+ unsigned bits);
+/**
+ * tlb_flush_page_range_by_mmuidx_all_cpus_synced:
+ * @cpu: Originating CPU of the flush
+ * @addr: virtual address of start of page range to be flushed
+ * @length: the number of bytes to be flushed
+ * @idxmap: bitmap of MMU indexes to flush
+ *
+ * Flush a range of pages from the TLB of all CPUs, for the specified MMU
+ * indexes like tlb_flush_page_by_mmuidx_all_cpus except the source
+ * vCPUs work is scheduled as safe work meaning all flushes will be
+ * complete once the source vCPUs safe work is complete. This will
+ * depend on when the guests translation ends the TB.
+ */
+void tlb_flush_page_range_bits_by_mmuidx_all_cpus_synced(CPUState *cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits);
/**
* tlb_flush_by_mmuidx:
* @cpu: CPU whose TLB should be flushed
@@ -313,6 +344,21 @@ static inline void tlb_flush_page_all_cpus_synced(CPUState
*src,
target_ulong addr)
{
}
+static inline void tlb_flush_page_range_bits_by_mmuidx(CPUState *cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits)
+{
+}
+static inline void
+tlb_flush_page_range_bits_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
+ target_ulong addr,
+ target_ulong length,
+ uint16_t idxmap,
+ unsigned bits)
+{
+}
static inline void tlb_flush(CPUState *cpu)
{
}
--
2.26.2
