ARMv8.4 adds the mandatory FEAT_TLBIRANGE, which provides instructions
for invalidating ranges of entries.
Signed-off-by: Rebecca Cran <rebe...@nuviainc.com>
---
accel/tcg/cputlb.c | 24 ++
include/exec/exec-all.h | 39 +++
target/arm/helper.c | 273 ++++++++++++++++++++
3 files changed, 336 insertions(+)
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index 42ab79c1a582..103f363b42f3 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -603,6 +603,30 @@ void tlb_flush_page(CPUState *cpu, target_ulong addr)
tlb_flush_page_by_mmuidx(cpu, addr, ALL_MMUIDX_BITS);
}
+void tlb_flush_page_range_by_mmuidx(CPUState *cpu, target_ulong addr,
+ int num_pages, uint16_t idxmap)
+{
+ int i;
+
+ for (i = 0; i < num_pages; i++) {
+ tlb_flush_page_by_mmuidx(cpu, addr + (i * TARGET_PAGE_SIZE), idxmap);
+ }
+}
+
+void tlb_flush_page_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
+ target_ulong addr,
+ int num_pages,
+ uint16_t idxmap)
+{
+ int i;
+
+ for (i = 0; i < num_pages; i++) {
+ tlb_flush_page_by_mmuidx_all_cpus_synced(src_cpu,
+ addr + (i * TARGET_PAGE_SIZE),
+ idxmap);
+ }
+}
+
void tlb_flush_page_by_mmuidx_all_cpus(CPUState *src_cpu, target_ulong addr,
uint16_t idxmap)
{
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 94fe05daaa18..c379e2404443 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -220,6 +220,35 @@ void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu,
target_ulong addr,
*/
void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr,
uint16_t idxmap);
+/**
+ * tlb_flush_page_range_by_mmuidx:
+ * @cpu: CPU whose TLB should be flushed
+ * @addr: virtual address of start of page range to be flushed
+ * @num_pages: the number of pages to be flushed
+ * @idxmap: bitmap of MMU indexes to flush
+ *
+ * Flush a range of pages from the TLB of the specified CPU, for the specified
+ * MMU indexes.
+ */
+void tlb_flush_page_range_by_mmuidx(CPUState *cpu, target_ulong addr,
+ int num_pages, uint16_t idxmap);
+/**
+ * tlb_flush_page_range_by_mmuidx_all_cpus_synced:
+ * @cpu: Originating CPU of the flush
+ * @addr: virtual address of start of page range to be flushed
+ * @num_pages: the number of pages to be flushed
+ * @idxmap: bitmap of MMU indexes to flush
+ *
+ * Flush a range of pages from the TLB of all CPUs, for the specified MMU
+ * indexes like tlb_flush_page_by_mmuidx_all_cpus except the source
+ * vCPUs work is scheduled as safe work meaning all flushes will be
+ * complete once the source vCPUs safe work is complete. This will
+ * depend on when the guests translation ends the TB.
+ */
+void tlb_flush_page_range_by_mmuidx_all_cpus_synced(CPUState *cpu,
+ target_ulong addr,
+ int num_pages,
+ uint16_t idxmap);
/**
* tlb_flush_by_mmuidx:
* @cpu: CPU whose TLB should be flushed
@@ -321,6 +350,16 @@ static inline void tlb_flush_page_all_cpus_synced(CPUState
*src,
target_ulong addr)
{
}
+static void tlb_flush_page_range_by_mmuidx(CPUState *cpu, target_ulong addr,
+ int num_pages, int idxmap)
+{
+}
+static void tlb_flush_page_range_by_mmuidx_all_cpus_synced(CPUState *src_cpu,
+ target_ulong addr,
+ int num_pages,
+ uint16_t idxmap)
+{
+}
static inline void tlb_flush(CPUState *cpu)
{
}
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 28556eb48b44..fffac1e1efa9 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -4667,6 +4667,165 @@ static void tlbi_aa64_vae3is_write(CPUARMState *env,
const ARMCPRegInfo *ri,
ARMMMUIdxBit_SE3, bits);
}
+static int tlbi_aa64_range_get_num_pages(CPUARMState *env, uint64_t value,
+ uint64_t addr)
+{
+ uint64_t exponent;
+ uint64_t high_addr;
+ int page_size;
+ int page_shift;
+ uint64_t scale = (value >> 44) & 0x3;
+ uint64_t num = (value >> 39) & 0xF;
+ int page_size_granule = (value >> 46) & 0x3;
+
+ switch (page_size_granule) {
+ case 1:
+ page_size = 4096;
+ page_shift = 12;
+ break;
+ case 2:
+ page_size = 16384;
+ page_shift = 14;
+ break;
+ case 3:
+ page_size = 65536;
+ page_shift = 16;
+ break;
+ default:
+ qemu_log_mask(LOG_GUEST_ERROR, "Invalid page size granule %d\n",
+ page_size_granule);
+
+ raise_exception(env, EXCP_UDEF, syn_uncategorized(),
+ exception_target_el(env));
+
+ break;
+ }
+
+ exponent = (5 * scale) + 1;
+ high_addr = addr + (((num + 1) << exponent) * page_size);
+
+ return (high_addr - addr) >> page_shift;
+}
+
+
+static void tlbi_aa64_rvae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, EL1&0.
+ * Currently handles all of RVAE1, RVAAE1, RVAALE1 and RVALE1,
+ * since we don't support flush-for-specific-ASID-only or
+ * flush-last-level-only.
+ */
+ CPUState *cs = env_cpu(env);
+ int mask = vae1_tlbmask(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ if (tlb_force_broadcast(env)) {
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages,
+ mask);
+ } else {
+ tlb_flush_page_range_by_mmuidx(cs, addr, num_pages, mask);
+ }
+}
+
+static void tlbi_aa64_rvae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, Inner/Outer Shareable EL1&0.
+ * Currently handles all of RVAE1IS, RVAE1OS, RVAAE1IS, RVAAE1OS,
+ * RVAALE1IS, RVAALE1OS, RVALE1IS and RVALE1OS, since we don't support
+ * flush-for-specific-ASID-only, flush-last-level-only or inner/outer
+ * shareable specific flushes.
+ */
+ CPUState *cs = env_cpu(env);
+ int mask = vae1_tlbmask(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages, mask);
+}
+
+static void tlbi_aa64_rvae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, EL2.
+ * Currently handles all of RVAE2, RVAAE2, RVAALE2 and RVALE2,
+ * since we don't support flush-for-specific-ASID-only or
+ * flush-last-level-only.
+ */
+ CPUState *cs = env_cpu(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ if (tlb_force_broadcast(env)) {
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages,
+ ARMMMUIdxBit_E2);
+ } else {
+ tlb_flush_page_range_by_mmuidx(cs, addr, num_pages, ARMMMUIdxBit_E2);
+ }
+}
+
+static void tlbi_aa64_rvae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, Inner/Outer Shareable, EL2.
+ * Currently handles all of RVAE2IS, RVAE2OS, RVAAE2IS, RVAAE2OS,
+ * RVAALE2IS, RVAALE2OS, RVALE2IS and RVALE2OS, since we don't support
+ * flush-for-specific-ASID-only, flush-last-level-only or inner/outer
+ * shareable specific flushes.
+ */
+ CPUState *cs = env_cpu(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages,
+ ARMMMUIdxBit_E2);
+}
+
+static void tlbi_aa64_rvae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, EL3.
+ * Currently handles all of RVAE3, RVAAE3, RVAALE3 and RVALE3,
+ * since we don't support flush-for-specific-ASID-only or
+ * flush-last-level-only.
+ */
+ CPUState *cs = env_cpu(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ if (tlb_force_broadcast(env)) {
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages,
+ ARMMMUIdxBit_SE3);
+ } else {
+ tlb_flush_page_range_by_mmuidx(cs, addr, num_pages, ARMMMUIdxBit_SE3);
+ }
+}
+
+static void tlbi_aa64_rvae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /*
+ * Invalidate by VA range, EL3, Inner/Outer Shareable.
+ * Currently handles all of RVAE3IS, RVAE3OS, RVAAE3IS, RVAAE3OS,
+ * RVAALE3IS, RVAALE3OS, RVALE3IS, and RVALE3OS, since we don't support
+ * flush-for-specific-ASID-only, flush-last-level-only or inner/outer
+ * specific flushes.
+ */
+ CPUState *cs = env_cpu(env);
+ uint64_t addr = (value & 0xFFFFFFFFFUL) << TARGET_PAGE_BITS;
+ int num_pages = tlbi_aa64_range_get_num_pages(env, value, addr);
+
+ tlb_flush_page_range_by_mmuidx_all_cpus_synced(cs, addr, num_pages,
+ ARMMMUIdxBit_SE3);
+}
+
static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
bool isread)
{
@@ -4938,6 +5097,63 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
{ .name = "TLBI_IPAS2LE1OS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 4,
.access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RIPAS2E1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 2,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RIPAS2E1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 2,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RIPAS2E1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 3,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RVAAE1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 3,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1_write },
+ { .name = "TLBI_RVAAE1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 3,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVAAE1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 3,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVAE1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1_write },
+ { .name = "TLBI_RVAE1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 1,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVAE1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
+ .access = PL1_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVAE2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 1,
+ .access = PL2_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2_write },
+ { .name = "TLBI_RVAE2IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 1,
+ .access = PL2_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2is_write },
+ { .name = "TLBI_RVAE2OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 1,
+ .access = PL2_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2is_write },
+ { .name = "TLBI_RVAE3", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 1,
+ .access = PL3_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3_write },
+ { .name = "TLBI_RVAE3IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 1,
+ .access = PL3_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3is_write },
+ { .name = "TLBI_RVAE3OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 1,
+ .access = PL3_W, .accessfn = access_ttlb, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3is_write },
{ .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
.opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
.access = PL2_W, .type = ARM_CP_NO_RAW,
@@ -5046,6 +5262,63 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
{ .name = "TLBIIPAS2LIS",
.cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
.type = ARM_CP_NOP, .access = PL2_W },
+ { .name = "TLBI_RIPAS2LE1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 6,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RIPAS2LE1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 6,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RIPAS2LE1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 7,
+ .access = PL2_W, .type = ARM_CP_NOP },
+ { .name = "TLBI_RVAALE1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 7,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1_write },
+ { .name = "TLBI_RVAALE1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 7,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVAALE1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 7,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVALE1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 5,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1_write },
+ { .name = "TLBI_RVALE1IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 2, .opc2 = 5,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVALE1OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 5,
+ .access = PL1_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae1is_write },
+ { .name = "TLBI_RVALE2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 6, .opc2 = 5,
+ .access = PL2_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2_write },
+ { .name = "TLBI_RVALE2IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 2, .opc2 = 5,
+ .access = PL2_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2is_write },
+ { .name = "TLBI_RVALE2OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 5, .opc2 = 5,
+ .access = PL2_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae2is_write },
+ { .name = "TLBI_RVALE3", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 6, .opc2 = 5,
+ .access = PL3_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3_write },
+ { .name = "TLBI_RVALE3IS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 2, .opc2 = 5,
+ .access = PL3_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3is_write },
+ { .name = "TLBI_RVALE3OS", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 5, .opc2 = 5,
+ .access = PL3_W, .type = ARM_CP_NO_RAW,
+ .writefn = tlbi_aa64_rvae3is_write },
/* 32 bit cache operations */
{ .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
.type = ARM_CP_NOP, .access = PL1_W, .accessfn = aa64_cacheop_pou_access
},
--
2.26.2
