The purpose of this new bitmap is to flag the memory pages that are in
the middle of LL/SC operations (after a LL, before a SC) on a per-vCPU
basis.
For all these pages, the corresponding TLB entries will be generated
in such a way to force the slow-path if at least one vCPU has the bit
not set.
When the system starts, the whole memory is dirty (all the bitmap is
set). A page, after being marked as exclusively-clean, will be
restored as dirty after the SC.
The accessors to this bitmap are currently not atomic, but they have to
be so in a real multi-threading TCG.
Suggested-by: Jani Kokkonen <jani.kokko...@huawei.com>
Suggested-by: Claudio Fontana <claudio.font...@huawei.com>
Signed-off-by: Alvise Rigo <a.r...@virtualopensystems.com>
---
exec.c | 7 +++--
include/exec/memory.h | 3 ++-
include/exec/ram_addr.h | 68 +++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 75 insertions(+), 3 deletions(-)
diff --git a/exec.c b/exec.c
index 7d60e15..f113076 100644
--- a/exec.c
+++ b/exec.c
@@ -1493,11 +1493,14 @@ static ram_addr_t ram_block_add(RAMBlock *new_block,
Error **errp)
int i;
/* ram_list.dirty_memory[] is protected by the iothread lock. */
- for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+ for (i = 0; i < DIRTY_MEMORY_EXCLUSIVE; i++) {
ram_list.dirty_memory[i] =
bitmap_zero_extend(ram_list.dirty_memory[i],
old_ram_size, new_ram_size);
- }
+ }
+ ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE] = bitmap_zero_extend(
+ ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE],
+ old_ram_size * smp_cpus, new_ram_size * smp_cpus);
}
cpu_physical_memory_set_dirty_range(new_block->offset,
new_block->used_length,
diff --git a/include/exec/memory.h b/include/exec/memory.h
index 1394715..a525259 100644
--- a/include/exec/memory.h
+++ b/include/exec/memory.h
@@ -19,7 +19,8 @@
#define DIRTY_MEMORY_VGA 0
#define DIRTY_MEMORY_CODE 1
#define DIRTY_MEMORY_MIGRATION 2
-#define DIRTY_MEMORY_NUM 3 /* num of dirty bits */
+#define DIRTY_MEMORY_EXCLUSIVE 3
+#define DIRTY_MEMORY_NUM 4 /* num of dirty bits */
#include <stdint.h>
#include <stdbool.h>
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index c113f21..6b678d6 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -21,6 +21,7 @@
#ifndef CONFIG_USER_ONLY
#include "hw/xen/xen.h"
+#include "sysemu/sysemu.h"
ram_addr_t qemu_ram_alloc_from_file(ram_addr_t size, MemoryRegion *mr,
bool share, const char *mem_path,
@@ -135,6 +136,10 @@ static inline void
cpu_physical_memory_set_dirty_range(ram_addr_t start,
if (unlikely(mask & (1 << DIRTY_MEMORY_CODE))) {
bitmap_set_atomic(d[DIRTY_MEMORY_CODE], page, end - page);
}
+ if (unlikely(mask & (1 << DIRTY_MEMORY_EXCLUSIVE))) {
+ bitmap_set_atomic(d[DIRTY_MEMORY_EXCLUSIVE],
+ page * smp_cpus, (end - page) * smp_cpus);
+ }
xen_modified_memory(start, length);
}
@@ -249,5 +254,68 @@ uint64_t cpu_physical_memory_sync_dirty_bitmap(unsigned
long *dest,
return num_dirty;
}
+/* Exclusive bitmap support. */
+#define EXCL_BITMAP_GET_OFFSET(addr) (smp_cpus * (addr >> TARGET_PAGE_BITS))
+static inline void cpu_physical_memory_set_excl_dirty(ram_addr_t addr,
+ uint32_t cpu_index)
+{
+ set_bit_atomic(EXCL_BITMAP_GET_OFFSET(addr) + cpu_index,
+ ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE]);
+}
+
+static inline int cpu_physical_memory_excl_atleast_one_clean(ram_addr_t addr)
+{
+ unsigned long *bitmap = ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE];
+ unsigned long next, end;
+
+ if (likely(smp_cpus <= BITS_PER_LONG)) {
+ unsigned long mask = (1 << smp_cpus) - 1;
+
+ return
+ (mask & (bitmap[BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))] >>
+ (EXCL_BITMAP_GET_OFFSET(addr) & (BITS_PER_LONG-1)))) != mask;
+ }
+
+ end = BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)) + smp_cpus;
+ next = find_next_zero_bit(bitmap, end,
+ BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)));
+
+ return next < end;
+}
+
+static inline int cpu_physical_memory_excl_is_dirty(ram_addr_t addr,
+ unsigned long cpu)
+{
+ unsigned long *bitmap = ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE];
+ unsigned long end, next;
+ uint32_t add;
+
+ assert(cpu <= smp_cpus);
+
+ if (likely(smp_cpus <= BITS_PER_LONG)) {
+ cpu = (cpu == smp_cpus) ? (1 << cpu) - 1 : (1 << cpu);
+
+ return cpu & (bitmap[BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))] >>
+ (EXCL_BITMAP_GET_OFFSET(addr) & (BITS_PER_LONG-1)));
+ }
+
+ add = (cpu == smp_cpus) ? 0 : 1;
+ end = BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr)) + cpu + add;
+ next = find_next_bit(bitmap, end, BIT_WORD(EXCL_BITMAP_GET_OFFSET(addr))
+ + (cpu % smp_cpus));
+
+ return next < end;
+}
+
+static inline bool cpu_physical_memory_clear_excl_dirty(ram_addr_t addr,
+ uint32_t cpu_index)
+{
+ return bitmap_test_and_clear_atomic(
+ ram_list.dirty_memory[DIRTY_MEMORY_EXCLUSIVE],
+ EXCL_BITMAP_GET_OFFSET(addr) + cpu_index, 1);
+}
+
+
+
#endif
#endif
--
2.5.0
