RAMList: Upstream dirty tracking implementation.
This modifies the implementation of dirty RAM tracking to follow
the upstream implementation, which uses 3 parallel bitmap arrays
instead of a single one merging all components.
Change-Id: I75c418444310f159973840aa765db65294859702
diff --git a/Makefile.target b/Makefile.target
index e68de28..ca3bd44 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -311,6 +311,8 @@
android/opengles.c \
android/user-events-qemu.c \
hw/core/loader.c \
+ util/bitmap.c \
+ util/bitops.c \
ui/keymaps.c \
util/qemu-timer-common.c \
util/iov.c \
diff --git a/arch_init.c b/arch_init.c
index 16d6690..cfd0fad 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -126,13 +126,14 @@
current_addr = block->offset + offset;
do {
- if (cpu_physical_memory_get_dirty(current_addr, MIGRATION_DIRTY_FLAG)) {
+ if (cpu_physical_memory_get_dirty(current_addr, TARGET_PAGE_SIZE,
+ DIRTY_MEMORY_MIGRATION)) {
uint8_t *p;
int cont = (block == last_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
cpu_physical_memory_reset_dirty(current_addr,
- current_addr + TARGET_PAGE_SIZE,
- MIGRATION_DIRTY_FLAG);
+ TARGET_PAGE_SIZE,
+ DIRTY_MEMORY_MIGRATION);
p = block->host + offset;
@@ -188,7 +189,8 @@
ram_addr_t addr;
for (addr = block->offset; addr < block->offset + block->length;
addr += TARGET_PAGE_SIZE) {
- if (cpu_physical_memory_get_dirty(addr, MIGRATION_DIRTY_FLAG)) {
+ if (cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
+ DIRTY_MEMORY_MIGRATION)) {
count++;
}
}
@@ -279,9 +281,10 @@
QTAILQ_FOREACH(block, &ram_list.blocks, next) {
for (addr = block->offset; addr < block->offset + block->length;
addr += TARGET_PAGE_SIZE) {
- if (!cpu_physical_memory_get_dirty(addr,
- MIGRATION_DIRTY_FLAG)) {
- cpu_physical_memory_set_dirty(addr);
+ if (!cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
+ DIRTY_MEMORY_MIGRATION)) {
+ cpu_physical_memory_set_dirty_flag(
+ addr, DIRTY_MEMORY_MIGRATION);
}
}
}
diff --git a/cputlb.c b/cputlb.c
index c721f58..a999d49 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -118,9 +118,8 @@
can be detected */
void tlb_protect_code(ram_addr_t ram_addr)
{
- cpu_physical_memory_reset_dirty(ram_addr,
- ram_addr + TARGET_PAGE_SIZE,
- CODE_DIRTY_FLAG);
+ cpu_physical_memory_reset_dirty(ram_addr, TARGET_PAGE_SIZE,
+ DIRTY_MEMORY_CODE);
}
/* update the TLB so that writes in physical page 'phys_addr' are no longer
@@ -128,7 +127,7 @@
void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr,
target_ulong vaddr)
{
- cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
}
static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
@@ -150,6 +149,26 @@
}
}
+void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length)
+{
+ CPUState *cpu;
+ CPUArchState *env;
+
+ CPU_FOREACH(cpu) {
+ int mmu_idx;
+
+ env = cpu->env_ptr;
+ for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
+ unsigned int i;
+
+ for (i = 0; i < CPU_TLB_SIZE; i++) {
+ tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
+ start1, length);
+ }
+ }
+ }
+}
+
static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
{
if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
@@ -288,7 +307,7 @@
/* Write access calls the I/O callback. */
te->addr_write = address | TLB_MMIO;
} else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
- !cpu_physical_memory_is_dirty(pd)) {
+ cpu_physical_memory_is_clean(pd)) {
te->addr_write = address | TLB_NOTDIRTY;
} else {
te->addr_write = address;
diff --git a/exec.c b/exec.c
index 4818eaf..ef45e00 100644
--- a/exec.c
+++ b/exec.c
@@ -39,6 +39,7 @@
#include "hw/hw.h"
#include "hw/qdev.h"
#include "hw/xen/xen.h"
+#include "qemu/bitmap.h"
#include "qemu/osdep.h"
#include "qemu/tls.h"
#include "sysemu/kvm.h"
@@ -49,6 +50,7 @@
#if defined(CONFIG_USER_ONLY)
#include <qemu.h>
#endif
+#include "translate-all.h"
//#define DEBUG_SUBPAGE
@@ -532,41 +534,28 @@
return block;
}
-/* Note: start and end must be within the same ram block. */
-void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
- int dirty_flags)
+static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
{
- unsigned long length, start1;
- int i;
+ ram_addr_t end = TARGET_PAGE_ALIGN(start + length);
start &= TARGET_PAGE_MASK;
- end = TARGET_PAGE_ALIGN(end);
- length = end - start;
+ RAMBlock* block = qemu_get_ram_block(start);
+ assert(block == qemu_get_ram_block(end - 1));
+ uintptr_t start1 = (uintptr_t)block->host + (start - block->offset);
+ cpu_tlb_reset_dirty_all(start1, length);
+}
+
+/* Note: start and end must be within the same ram block. */
+void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
+ unsigned client)
+{
if (length == 0)
return;
- cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
+ cpu_physical_memory_clear_dirty_range(start, length, client);
- /* we modify the TLB cache so that the dirty bit will be set again
- when accessing the range */
- start1 = (unsigned long)qemu_safe_ram_ptr(start);
- /* Chek that we don't span multiple blocks - this breaks the
- address comparisons below. */
- if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
- != (end - 1) - start) {
- abort();
- }
-
- CPUState *cpu;
- CPU_FOREACH(cpu) {
- int mmu_idx;
- for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
- for(i = 0; i < CPU_TLB_SIZE; i++) {
- CPUArchState* env = cpu->env_ptr;
- tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
- start1, length);
- }
- }
+ if (tcg_enabled()) {
+ tlb_reset_dirty_range_all(start, length);
}
}
@@ -603,7 +592,7 @@
p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
+ tlb_entry->addend);
ram_addr = qemu_ram_addr_from_host_nofail(p);
- if (!cpu_physical_memory_is_dirty(ram_addr)) {
+ if (cpu_physical_memory_is_clean(ram_addr)) {
tlb_entry->addr_write |= TLB_NOTDIRTY;
}
}
@@ -1079,6 +1068,9 @@
ram_addr_t size, void *host)
{
RAMBlock *block, *new_block;
+ ram_addr_t old_ram_size, new_ram_size;
+
+ old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
size = TARGET_PAGE_ALIGN(size);
new_block = g_malloc0(sizeof(*new_block));
@@ -1166,11 +1158,17 @@
ram_list.version++;
qemu_mutex_unlock_ramlist();
- ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
- last_ram_offset() >> TARGET_PAGE_BITS);
- memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
- 0xff, size >> TARGET_PAGE_BITS);
- //cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
+ new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
+
+ if (new_ram_size > old_ram_size) {
+ int i;
+ for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
+ ram_list.dirty_memory[i] =
+ bitmap_zero_extend(ram_list.dirty_memory[i],
+ old_ram_size, new_ram_size);
+ }
+ }
+ cpu_physical_memory_set_dirty_range(new_block->offset, size);
qemu_ram_setup_dump(new_block->host, size);
//qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
@@ -1463,61 +1461,52 @@
static void notdirty_mem_writeb(void *opaque, hwaddr ram_addr,
uint32_t val)
{
- int dirty_flags;
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
- if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
+ if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
tb_invalidate_phys_page_fast0(ram_addr, 1);
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
}
stb_p(qemu_get_ram_ptr(ram_addr), val);
- dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
- cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
/* we remove the notdirty callback only if the code has been
flushed */
- if (dirty_flags == 0xff)
- tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+ if (!cpu_physical_memory_is_clean(ram_addr)) {
+ CPUArchState *env = current_cpu->env_ptr;
+ tlb_set_dirty(env, env->mem_io_vaddr);
+ }
}
static void notdirty_mem_writew(void *opaque, hwaddr ram_addr,
uint32_t val)
{
- int dirty_flags;
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
- if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
- tb_invalidate_phys_page_fast0(ram_addr, 2);
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
+ if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+ tb_invalidate_phys_page_fast0(ram_addr, 1);
}
stw_p(qemu_get_ram_ptr(ram_addr), val);
- dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
- cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
/* we remove the notdirty callback only if the code has been
flushed */
- if (dirty_flags == 0xff)
- tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+ if (!cpu_physical_memory_is_clean(ram_addr)) {
+ CPUArchState *env = current_cpu->env_ptr;
+ tlb_set_dirty(env, env->mem_io_vaddr);
+ }
}
static void notdirty_mem_writel(void *opaque, hwaddr ram_addr,
uint32_t val)
{
- int dirty_flags;
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
- if (!(dirty_flags & CODE_DIRTY_FLAG)) {
-#if !defined(CONFIG_USER_ONLY)
- tb_invalidate_phys_page_fast0(ram_addr, 4);
- dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
-#endif
+ if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
+ tb_invalidate_phys_page_fast0(ram_addr, 1);
}
stl_p(qemu_get_ram_ptr(ram_addr), val);
- dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
- cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
+ cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
/* we remove the notdirty callback only if the code has been
flushed */
- if (dirty_flags == 0xff)
- tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
+ if (!cpu_physical_memory_is_clean(ram_addr)) {
+ CPUArchState *env = current_cpu->env_ptr;
+ tlb_set_dirty(env, env->mem_io_vaddr);
+ }
}
static CPUReadMemoryFunc * const error_mem_read[3] = {
@@ -1532,16 +1521,6 @@
notdirty_mem_writel,
};
-static void tb_check_watchpoint(CPUArchState* env)
-{
- TranslationBlock *tb = tb_find_pc(env->mem_io_pc);
- if (!tb) {
- cpu_abort(env, "check_watchpoint: could not find TB for "
- "pc=%p", (void *)env->mem_io_pc);
- }
- cpu_restore_state(env, env->mem_io_pc);
- tb_phys_invalidate(tb, -1);
-}
/* Generate a debug exception if a watchpoint has been hit. */
static void check_watchpoint(int offset, int len_mask, int flags)
@@ -1919,11 +1898,12 @@
static void invalidate_and_set_dirty(hwaddr addr,
hwaddr length)
{
- if (!cpu_physical_memory_is_dirty(addr)) {
+ if (cpu_physical_memory_is_clean(addr)) {
/* invalidate code */
tb_invalidate_phys_page_range(addr, addr + length, 0);
/* set dirty bit */
- cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
+ cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
+ cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
}
}
@@ -2435,12 +2415,13 @@
stl_p(ptr, val);
if (unlikely(in_migration)) {
- if (!cpu_physical_memory_is_dirty(addr1)) {
+ if (cpu_physical_memory_is_clean(addr1)) {
/* invalidate code */
tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
/* set dirty bit */
- cpu_physical_memory_set_dirty_flags(
- addr1, (0xff & ~CODE_DIRTY_FLAG));
+ cpu_physical_memory_set_dirty_flag(addr1,
+ DIRTY_MEMORY_MIGRATION);
+ cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
}
}
}
@@ -2596,12 +2577,12 @@
stw_p(ptr, val);
break;
}
- if (!cpu_physical_memory_is_dirty(addr1)) {
+ if (cpu_physical_memory_is_clean(addr1)) {
/* invalidate code */
tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
/* set dirty bit */
- cpu_physical_memory_set_dirty_flags(addr1,
- (0xff & ~CODE_DIRTY_FLAG));
+ cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_MIGRATION);
+ cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
}
}
}
diff --git a/hw/android/goldfish/fb.c b/hw/android/goldfish/fb.c
index 5fb3215..bbb392a 100644
--- a/hw/android/goldfish/fb.c
+++ b/hw/android/goldfish/fb.c
@@ -292,20 +292,9 @@
* changed pixels.
*/
if (dirty_addr != 0) {
- int dirty = 0;
- int len = fbs->src_pitch;
-
- while (len > 0) {
- int len2 = TARGET_PAGE_SIZE - (dirty_addr & (TARGET_PAGE_SIZE-1));
-
- if (len2 > len)
- len2 = len;
-
- dirty |= cpu_physical_memory_get_dirty(dirty_addr, VGA_DIRTY_FLAG);
- dirty_addr += len2;
- len -= len2;
- }
-
+ int dirty = cpu_physical_memory_get_dirty(dirty_addr,
+ fbs->src_pitch,
+ DIRTY_MEMORY_VGA);
if (!dirty) { /* this line was not modified, skip to next one */
goto NEXT_LINE;
}
@@ -445,8 +434,8 @@
/* Always clear the dirty VGA bits */
cpu_physical_memory_reset_dirty(dirty_base + rect->ymin * fbs->src_pitch,
- dirty_base + (rect->ymax+1)* fbs->src_pitch,
- VGA_DIRTY_FLAG);
+ (rect->ymax - rect->ymin + 1) * fbs->src_pitch,
+ DIRTY_MEMORY_VGA);
return 1;
}
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index cd04011..1a7b2ab 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -470,9 +470,14 @@
int fd;
} RAMBlock;
+#define DIRTY_MEMORY_VGA 0
+#define DIRTY_MEMORY_CODE 1
+#define DIRTY_MEMORY_MIGRATION 2
+#define DIRTY_MEMORY_NUM 3 /* num of dirty bits */
+
typedef struct RAMList {
QemuMutex mutex;
- uint8_t *phys_dirty;
+ unsigned long *dirty_memory[DIRTY_MEMORY_NUM];
RAMBlock *mru_block;
QTAILQ_HEAD(ram, RAMBlock) blocks;
uint32_t version;
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index fd4e834..0b4ea36 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -20,7 +20,10 @@
#define RAM_ADDR_H
#ifndef CONFIG_USER_ONLY
+#include "exec/cpu-all.h"
#include "hw/xen/xen.h"
+#include "qemu/bitmap.h"
+#include "qemu/bitops.h"
ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
ram_addr_t size, void *host);
@@ -37,48 +40,122 @@
void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr);
-static inline int cpu_physical_memory_get_dirty(ram_addr_t addr,
- int dirty_flags)
+static inline int cpu_physical_memory_get_dirty(ram_addr_t start,
+ ram_addr_t length,
+ unsigned client)
{
- return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] & dirty_flags;
+ unsigned long end, page, next;
+
+ assert(client < DIRTY_MEMORY_NUM);
+
+ end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+ page = start >> TARGET_PAGE_BITS;
+ next = find_next_bit(ram_list.dirty_memory[client], end, page);
+
+ return next < end;
}
-static inline int cpu_physical_memory_get_dirty_flags(ram_addr_t addr)
+static inline bool cpu_physical_memory_get_dirty_flag(ram_addr_t addr,
+ unsigned client)
{
- return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS];
+ return cpu_physical_memory_get_dirty(addr, 1, client);
}
-/* read dirty bit (return 0 or 1) */
-static inline int cpu_physical_memory_is_dirty(ram_addr_t addr)
+static inline bool cpu_physical_memory_is_clean(ram_addr_t addr)
{
- return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] == 0xff;
+ bool vga = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_VGA);
+ bool code = cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_CODE);
+ bool migration =
+ cpu_physical_memory_get_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
+ return !(vga && code && migration);
}
-static inline void cpu_physical_memory_set_dirty(ram_addr_t addr)
+static inline void cpu_physical_memory_set_dirty_flag(ram_addr_t addr,
+ unsigned client)
{
- ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] = 0xff;
+ assert(client < DIRTY_MEMORY_NUM);
+ set_bit(addr >> TARGET_PAGE_BITS, ram_list.dirty_memory[client]);
}
-static inline int cpu_physical_memory_set_dirty_flags(ram_addr_t addr,
- int dirty_flags)
+static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
+ ram_addr_t length)
{
- return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] |= dirty_flags;
+ unsigned long end, page;
+
+ end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+ page = start >> TARGET_PAGE_BITS;
+ bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION], page, end - page);
+ bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_VGA], page, end - page);
+ bitmap_set(ram_list.dirty_memory[DIRTY_MEMORY_CODE], page, end - page);
+ //xen_modified_memory(start, length);
}
-static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
- int length,
- int dirty_flags)
+#if !defined(_WIN32)
+static inline void cpu_physical_memory_set_dirty_lebitmap(unsigned long *bitmap,
+ ram_addr_t start,
+ ram_addr_t pages)
{
- int i, mask, len;
- uint8_t *p;
+ unsigned long i, j;
+ unsigned long page_number, c;
+ hwaddr addr;
+ ram_addr_t ram_addr;
+ unsigned long len = (pages + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
+ unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
+ unsigned long page = BIT_WORD(start >> TARGET_PAGE_BITS);
- len = length >> TARGET_PAGE_BITS;
- mask = ~dirty_flags;
- p = ram_list.phys_dirty + (start >> TARGET_PAGE_BITS);
- for (i = 0; i < len; i++) {
- p[i] &= mask;
+ /* start address is aligned at the start of a word? */
+ if ((((page * BITS_PER_LONG) << TARGET_PAGE_BITS) == start) &&
+ (hpratio == 1)) {
+ long k;
+ long nr = BITS_TO_LONGS(pages);
+
+ for (k = 0; k < nr; k++) {
+ if (bitmap[k]) {
+ unsigned long temp = leul_to_cpu(bitmap[k]);
+
+ ram_list.dirty_memory[DIRTY_MEMORY_MIGRATION][page + k] |= temp;
+ ram_list.dirty_memory[DIRTY_MEMORY_VGA][page + k] |= temp;
+ ram_list.dirty_memory[DIRTY_MEMORY_CODE][page + k] |= temp;
+ }
+ }
+ //xen_modified_memory(start, pages);
+ } else {
+ /*
+ * bitmap-traveling is faster than memory-traveling (for addr...)
+ * especially when most of the memory is not dirty.
+ */
+ for (i = 0; i < len; i++) {
+ if (bitmap[i] != 0) {
+ c = leul_to_cpu(bitmap[i]);
+ do {
+ j = ffsl(c) - 1;
+ c &= ~(1ul << j);
+ page_number = (i * HOST_LONG_BITS + j) * hpratio;
+ addr = page_number * TARGET_PAGE_SIZE;
+ ram_addr = start + addr;
+ cpu_physical_memory_set_dirty_range(ram_addr,
+ TARGET_PAGE_SIZE * hpratio);
+ } while (c != 0);
+ }
+ }
}
}
+#endif /* not _WIN32 */
+
+static inline void cpu_physical_memory_clear_dirty_range(ram_addr_t start,
+ ram_addr_t length,
+ unsigned client)
+{
+ unsigned long end, page;
+
+ assert(client < DIRTY_MEMORY_NUM);
+ end = TARGET_PAGE_ALIGN(start + length) >> TARGET_PAGE_BITS;
+ page = start >> TARGET_PAGE_BITS;
+ bitmap_clear(ram_list.dirty_memory[client], page, end - page);
+}
+
+void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
+ unsigned client);
int cpu_physical_memory_set_dirty_tracking(int enable);
@@ -87,8 +164,5 @@
int cpu_physical_sync_dirty_bitmap(hwaddr start_addr,
hwaddr end_addr);
-void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
- int dirty_flags);
-
#endif
#endif
\ No newline at end of file
diff --git a/include/qemu/bitmap.h b/include/qemu/bitmap.h
index 308bbb7..1babd5d 100644
--- a/include/qemu/bitmap.h
+++ b/include/qemu/bitmap.h
@@ -31,7 +31,7 @@
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
* bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal?
- * bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap?
+ * bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap?
* bitmap_empty(src, nbits) Are all bits zero in *src?
* bitmap_full(src, nbits) Are all bits set in *src?
* bitmap_set(dst, pos, nbits) Set specified bit area
@@ -62,71 +62,71 @@
)
#define DECLARE_BITMAP(name,bits) \
- unsigned long name[BITS_TO_LONGS(bits)]
+ unsigned long name[BITS_TO_LONGS(bits)]
#define small_nbits(nbits) \
- ((nbits) <= BITS_PER_LONG)
+ ((nbits) <= BITS_PER_LONG)
-int slow_bitmap_empty(const unsigned long *bitmap, int bits);
-int slow_bitmap_full(const unsigned long *bitmap, int bits);
+int slow_bitmap_empty(const unsigned long *bitmap, long bits);
+int slow_bitmap_full(const unsigned long *bitmap, long bits);
int slow_bitmap_equal(const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
void slow_bitmap_complement(unsigned long *dst, const unsigned long *src,
- int bits);
+ long bits);
void slow_bitmap_shift_right(unsigned long *dst,
- const unsigned long *src, int shift, int bits);
+ const unsigned long *src, int shift, long bits);
void slow_bitmap_shift_left(unsigned long *dst,
- const unsigned long *src, int shift, int bits);
+ const unsigned long *src, int shift, long bits);
int slow_bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
void slow_bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
void slow_bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
int slow_bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
int slow_bitmap_intersects(const unsigned long *bitmap1,
- const unsigned long *bitmap2, int bits);
+ const unsigned long *bitmap2, long bits);
-static inline unsigned long *bitmap_new(int nbits)
+static inline unsigned long *bitmap_new(long nbits)
{
- int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+ long len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
return g_malloc0(len);
}
-static inline void bitmap_zero(unsigned long *dst, int nbits)
+static inline void bitmap_zero(unsigned long *dst, long nbits)
{
if (small_nbits(nbits)) {
*dst = 0UL;
} else {
- int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+ long len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
memset(dst, 0, len);
}
}
-static inline void bitmap_fill(unsigned long *dst, int nbits)
+static inline void bitmap_fill(unsigned long *dst, long nbits)
{
size_t nlongs = BITS_TO_LONGS(nbits);
if (!small_nbits(nbits)) {
- int len = (nlongs - 1) * sizeof(unsigned long);
+ long len = (nlongs - 1) * sizeof(unsigned long);
memset(dst, 0xff, len);
}
dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits);
}
static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
- int nbits)
+ long nbits)
{
if (small_nbits(nbits)) {
*dst = *src;
} else {
- int len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
+ long len = BITS_TO_LONGS(nbits) * sizeof(unsigned long);
memcpy(dst, src, len);
}
}
static inline int bitmap_and(unsigned long *dst, const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
return (*dst = *src1 & *src2) != 0;
@@ -135,7 +135,7 @@
}
static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
*dst = *src1 | *src2;
@@ -145,7 +145,7 @@
}
static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
*dst = *src1 ^ *src2;
@@ -155,7 +155,7 @@
}
static inline int bitmap_andnot(unsigned long *dst, const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
return (*dst = *src1 & ~(*src2)) != 0;
@@ -163,8 +163,9 @@
return slow_bitmap_andnot(dst, src1, src2, nbits);
}
-static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
- int nbits)
+static inline void bitmap_complement(unsigned long *dst,
+ const unsigned long *src,
+ long nbits)
{
if (small_nbits(nbits)) {
*dst = ~(*src) & BITMAP_LAST_WORD_MASK(nbits);
@@ -174,7 +175,7 @@
}
static inline int bitmap_equal(const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
return ! ((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
@@ -183,7 +184,7 @@
}
}
-static inline int bitmap_empty(const unsigned long *src, int nbits)
+static inline int bitmap_empty(const unsigned long *src, long nbits)
{
if (small_nbits(nbits)) {
return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
@@ -192,7 +193,7 @@
}
}
-static inline int bitmap_full(const unsigned long *src, int nbits)
+static inline int bitmap_full(const unsigned long *src, long nbits)
{
if (small_nbits(nbits)) {
return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
@@ -202,7 +203,7 @@
}
static inline int bitmap_intersects(const unsigned long *src1,
- const unsigned long *src2, int nbits)
+ const unsigned long *src2, long nbits)
{
if (small_nbits(nbits)) {
return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
@@ -211,12 +212,21 @@
}
}
-void bitmap_set(unsigned long *map, int i, int len);
-void bitmap_clear(unsigned long *map, int start, int nr);
+void bitmap_set(unsigned long *map, long i, long len);
+void bitmap_clear(unsigned long *map, long start, long nr);
unsigned long bitmap_find_next_zero_area(unsigned long *map,
- unsigned long size,
- unsigned long start,
- unsigned int nr,
- unsigned long align_mask);
+ unsigned long size,
+ unsigned long start,
+ unsigned long nr,
+ unsigned long align_mask);
+
+static inline unsigned long *bitmap_zero_extend(unsigned long *old,
+ long old_nbits, long new_nbits)
+{
+ long new_len = BITS_TO_LONGS(new_nbits) * sizeof(unsigned long);
+ unsigned long *new = g_realloc(old, new_len);
+ bitmap_clear(new, old_nbits, new_nbits - old_nbits);
+ return new;
+}
#endif /* BITMAP_H */
diff --git a/include/qemu/bitops.h b/include/qemu/bitops.h
index 304c90c..340b1e7 100644
--- a/include/qemu/bitops.h
+++ b/include/qemu/bitops.h
@@ -28,7 +28,7 @@
* @nr: the bit to set
* @addr: the address to start counting from
*/
-static inline void set_bit(int nr, unsigned long *addr)
+static inline void set_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -41,7 +41,7 @@
* @nr: Bit to clear
* @addr: Address to start counting from
*/
-static inline void clear_bit(int nr, unsigned long *addr)
+static inline void clear_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -54,7 +54,7 @@
* @nr: Bit to change
* @addr: Address to start counting from
*/
-static inline void change_bit(int nr, unsigned long *addr)
+static inline void change_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -67,7 +67,7 @@
* @nr: Bit to set
* @addr: Address to count from
*/
-static inline int test_and_set_bit(int nr, unsigned long *addr)
+static inline int test_and_set_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -82,7 +82,7 @@
* @nr: Bit to clear
* @addr: Address to count from
*/
-static inline int test_and_clear_bit(int nr, unsigned long *addr)
+static inline int test_and_clear_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -97,7 +97,7 @@
* @nr: Bit to change
* @addr: Address to count from
*/
-static inline int test_and_change_bit(int nr, unsigned long *addr)
+static inline int test_and_change_bit(long nr, unsigned long *addr)
{
unsigned long mask = BIT_MASK(nr);
unsigned long *p = addr + BIT_WORD(nr);
@@ -112,7 +112,7 @@
* @nr: bit number to test
* @addr: Address to start counting from
*/
-static inline int test_bit(int nr, const unsigned long *addr)
+static inline int test_bit(long nr, const unsigned long *addr)
{
return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1)));
}
diff --git a/kvm-all.c b/kvm-all.c
index a8bcca4..cd8ddd3 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -300,6 +300,8 @@
return 0;
}
+#define ALIGN(x, y) (((x)+(y)-1) & ~((y)-1))
+
/**
* kvm_physical_sync_dirty_bitmap - Grab dirty bitmap from kernel space
* This function updates qemu's dirty bitmap using cpu_physical_memory_set_dirty().
@@ -312,9 +314,7 @@
hwaddr end_addr)
{
KVMState *s = kvm_state;
- unsigned long size, allocated_size = 0;
- hwaddr phys_addr;
- ram_addr_t addr;
+ size_t size, allocated_size = 0;
KVMDirtyLog d;
KVMSlot *mem;
int ret = 0;
@@ -328,12 +328,13 @@
break;
}
- size = ((mem->memory_size >> TARGET_PAGE_BITS) + 7) / 8;
+ size = ALIGN(((mem->memory_size) >> TARGET_PAGE_BITS),
+ /*HOST_LONG_BITS*/ 64) / 8;
if (size > allocated_size) {
d.dirty_bitmap = g_realloc(d.dirty_bitmap, size);
allocated_size = size;
}
- memset(d.dirty_bitmap, 0, size);
+ memset(d.dirty_bitmap, 0, allocated_size);
d.slot = mem->slot;
@@ -343,19 +344,10 @@
break;
}
- for (phys_addr = mem->start_addr, addr = mem->phys_offset;
- phys_addr - mem->start_addr < mem->memory_size;
- phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
- unsigned long *bitmap = (unsigned long *)d.dirty_bitmap;
- unsigned nr = (phys_addr - mem->start_addr) >> TARGET_PAGE_BITS;
- unsigned word = nr / (sizeof(*bitmap) * 8);
- unsigned bit = nr % (sizeof(*bitmap) * 8);
-
- if ((bitmap[word] >> bit) & 1) {
- cpu_physical_memory_set_dirty(addr);
- }
- }
- start_addr = phys_addr;
+ cpu_physical_memory_set_dirty_lebitmap(d.dirty_bitmap,
+ mem->phys_offset,
+ mem->memory_size / getpagesize());
+ start_addr += mem->memory_size;
if (!start_addr) {
// Handle wrap-around, which happens when a slot is mapped
// at the end of the physical address space.
diff --git a/translate-all.c b/translate-all.c
index e4a5e2b..351d949 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -1404,6 +1404,7 @@
tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
}
#endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
+#endif // !CONFIG_ANDROID
void tb_check_watchpoint(CPUArchState *env)
{
@@ -1417,7 +1418,6 @@
cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
tb_phys_invalidate(tb, -1);
}
-#endif // !CONFIG_ANDROID
#ifndef CONFIG_USER_ONLY
/* mask must never be zero, except for A20 change call */
diff --git a/translate-all.h b/translate-all.h
index 167eb58..498db1b 100644
--- a/translate-all.h
+++ b/translate-all.h
@@ -19,6 +19,8 @@
#ifndef TRANSLATE_ALL_H
#define TRANSLATE_ALL_H
+#include <stdbool.h>
+
/* Size of the L2 (and L3, etc) page tables. */
#define L2_BITS 10
#define L2_SIZE (1 << L2_BITS)
@@ -27,6 +29,7 @@
(((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
/* translate-all.c */
+bool tcg_enabled(void);
void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len);
void cpu_unlink_tb(CPUOldState *cpu);
void tb_check_watchpoint(CPUArchState *env);
diff --git a/util/bitmap.c b/util/bitmap.c
new file mode 100644
index 0000000..9c6bb52
--- /dev/null
+++ b/util/bitmap.c
@@ -0,0 +1,256 @@
+/*
+ * Bitmap Module
+ *
+ * Stolen from linux/src/lib/bitmap.c
+ *
+ * Copyright (C) 2010 Corentin Chary
+ *
+ * This source code is licensed under the GNU General Public License,
+ * Version 2.
+ */
+
+#include "qemu/bitops.h"
+#include "qemu/bitmap.h"
+
+/*
+ * bitmaps provide an array of bits, implemented using an an
+ * array of unsigned longs. The number of valid bits in a
+ * given bitmap does _not_ need to be an exact multiple of
+ * BITS_PER_LONG.
+ *
+ * The possible unused bits in the last, partially used word
+ * of a bitmap are 'don't care'. The implementation makes
+ * no particular effort to keep them zero. It ensures that
+ * their value will not affect the results of any operation.
+ * The bitmap operations that return Boolean (bitmap_empty,
+ * for example) or scalar (bitmap_weight, for example) results
+ * carefully filter out these unused bits from impacting their
+ * results.
+ *
+ * These operations actually hold to a slightly stronger rule:
+ * if you don't input any bitmaps to these ops that have some
+ * unused bits set, then they won't output any set unused bits
+ * in output bitmaps.
+ *
+ * The byte ordering of bitmaps is more natural on little
+ * endian architectures.
+ */
+
+int slow_bitmap_empty(const unsigned long *bitmap, long bits)
+{
+ long k, lim = bits/BITS_PER_LONG;
+
+ for (k = 0; k < lim; ++k) {
+ if (bitmap[k]) {
+ return 0;
+ }
+ }
+ if (bits % BITS_PER_LONG) {
+ if (bitmap[k] & BITMAP_LAST_WORD_MASK(bits)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+int slow_bitmap_full(const unsigned long *bitmap, long bits)
+{
+ long k, lim = bits/BITS_PER_LONG;
+
+ for (k = 0; k < lim; ++k) {
+ if (~bitmap[k]) {
+ return 0;
+ }
+ }
+
+ if (bits % BITS_PER_LONG) {
+ if (~bitmap[k] & BITMAP_LAST_WORD_MASK(bits)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+int slow_bitmap_equal(const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k, lim = bits/BITS_PER_LONG;
+
+ for (k = 0; k < lim; ++k) {
+ if (bitmap1[k] != bitmap2[k]) {
+ return 0;
+ }
+ }
+
+ if (bits % BITS_PER_LONG) {
+ if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) {
+ return 0;
+ }
+ }
+
+ return 1;
+}
+
+void slow_bitmap_complement(unsigned long *dst, const unsigned long *src,
+ long bits)
+{
+ long k, lim = bits/BITS_PER_LONG;
+
+ for (k = 0; k < lim; ++k) {
+ dst[k] = ~src[k];
+ }
+
+ if (bits % BITS_PER_LONG) {
+ dst[k] = ~src[k] & BITMAP_LAST_WORD_MASK(bits);
+ }
+}
+
+int slow_bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k;
+ long nr = BITS_TO_LONGS(bits);
+ unsigned long result = 0;
+
+ for (k = 0; k < nr; k++) {
+ result |= (dst[k] = bitmap1[k] & bitmap2[k]);
+ }
+ return result != 0;
+}
+
+void slow_bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k;
+ long nr = BITS_TO_LONGS(bits);
+
+ for (k = 0; k < nr; k++) {
+ dst[k] = bitmap1[k] | bitmap2[k];
+ }
+}
+
+void slow_bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k;
+ long nr = BITS_TO_LONGS(bits);
+
+ for (k = 0; k < nr; k++) {
+ dst[k] = bitmap1[k] ^ bitmap2[k];
+ }
+}
+
+int slow_bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k;
+ long nr = BITS_TO_LONGS(bits);
+ unsigned long result = 0;
+
+ for (k = 0; k < nr; k++) {
+ result |= (dst[k] = bitmap1[k] & ~bitmap2[k]);
+ }
+ return result != 0;
+}
+
+#define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) % BITS_PER_LONG))
+
+void bitmap_set(unsigned long *map, long start, long nr)
+{
+ unsigned long *p = map + BIT_WORD(start);
+ const long size = start + nr;
+ int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
+ unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
+
+ while (nr - bits_to_set >= 0) {
+ *p |= mask_to_set;
+ nr -= bits_to_set;
+ bits_to_set = BITS_PER_LONG;
+ mask_to_set = ~0UL;
+ p++;
+ }
+ if (nr) {
+ mask_to_set &= BITMAP_LAST_WORD_MASK(size);
+ *p |= mask_to_set;
+ }
+}
+
+void bitmap_clear(unsigned long *map, long start, long nr)
+{
+ unsigned long *p = map + BIT_WORD(start);
+ const long size = start + nr;
+ int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
+ unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
+
+ while (nr - bits_to_clear >= 0) {
+ *p &= ~mask_to_clear;
+ nr -= bits_to_clear;
+ bits_to_clear = BITS_PER_LONG;
+ mask_to_clear = ~0UL;
+ p++;
+ }
+ if (nr) {
+ mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
+ *p &= ~mask_to_clear;
+ }
+}
+
+#define ALIGN_MASK(x,mask) (((x)+(mask))&~(mask))
+
+/**
+ * bitmap_find_next_zero_area - find a contiguous aligned zero area
+ * @map: The address to base the search on
+ * @size: The bitmap size in bits
+ * @start: The bitnumber to start searching at
+ * @nr: The number of zeroed bits we're looking for
+ * @align_mask: Alignment mask for zero area
+ *
+ * The @align_mask should be one less than a power of 2; the effect is that
+ * the bit offset of all zero areas this function finds is multiples of that
+ * power of 2. A @align_mask of 0 means no alignment is required.
+ */
+unsigned long bitmap_find_next_zero_area(unsigned long *map,
+ unsigned long size,
+ unsigned long start,
+ unsigned long nr,
+ unsigned long align_mask)
+{
+ unsigned long index, end, i;
+again:
+ index = find_next_zero_bit(map, size, start);
+
+ /* Align allocation */
+ index = ALIGN_MASK(index, align_mask);
+
+ end = index + nr;
+ if (end > size) {
+ return end;
+ }
+ i = find_next_bit(map, end, index);
+ if (i < end) {
+ start = i + 1;
+ goto again;
+ }
+ return index;
+}
+
+int slow_bitmap_intersects(const unsigned long *bitmap1,
+ const unsigned long *bitmap2, long bits)
+{
+ long k, lim = bits/BITS_PER_LONG;
+
+ for (k = 0; k < lim; ++k) {
+ if (bitmap1[k] & bitmap2[k]) {
+ return 1;
+ }
+ }
+
+ if (bits % BITS_PER_LONG) {
+ if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) {
+ return 1;
+ }
+ }
+ return 0;
+}
