blob: 8c81cffaa7121f3f063fc009b194b58c72e9d6bc [file] [log] [blame]
bellard54936002003-05-13 00:25:15 +00001/*
bellardfd6ce8f2003-05-14 19:00:11 +00002 * virtual page mapping and translated block handling
ths5fafdf22007-09-16 21:08:06 +00003 *
bellard54936002003-05-13 00:25:15 +00004 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
Blue Swirl8167ee82009-07-16 20:47:01 +000017 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
bellard54936002003-05-13 00:25:15 +000018 */
bellard67b915a2004-03-31 23:37:16 +000019#include "config.h"
bellardd5a8f072004-09-29 21:15:28 +000020#ifdef _WIN32
21#include <windows.h>
22#else
bellarda98d49b2004-11-14 16:22:05 +000023#include <sys/types.h>
bellardd5a8f072004-09-29 21:15:28 +000024#include <sys/mman.h>
25#endif
bellard54936002003-05-13 00:25:15 +000026
Stefan Weil055403b2010-10-22 23:03:32 +020027#include "qemu-common.h"
bellard6180a182003-09-30 21:04:53 +000028#include "cpu.h"
29#include "exec-all.h"
bellardb67d9a52008-05-23 09:57:34 +000030#include "tcg.h"
pbrookb3c77242008-06-30 16:31:04 +000031#include "hw/hw.h"
Alex Williamsoncc9e98c2010-06-25 11:09:43 -060032#include "hw/qdev.h"
aliguori74576192008-10-06 14:02:03 +000033#include "osdep.h"
aliguori7ba1e612008-11-05 16:04:33 +000034#include "kvm.h"
Jun Nakajima432d2682010-08-31 16:41:25 +010035#include "hw/xen.h"
Blue Swirl29e922b2010-03-29 19:24:00 +000036#include "qemu-timer.h"
pbrook53a59602006-03-25 19:31:22 +000037#if defined(CONFIG_USER_ONLY)
38#include <qemu.h>
Riku Voipiofd052bf2010-01-25 14:30:49 +020039#include <signal.h>
Juergen Lockf01576f2010-03-25 22:32:16 +010040#if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41#include <sys/param.h>
42#if __FreeBSD_version >= 700104
43#define HAVE_KINFO_GETVMMAP
44#define sigqueue sigqueue_freebsd /* avoid redefinition */
45#include <sys/time.h>
46#include <sys/proc.h>
47#include <machine/profile.h>
48#define _KERNEL
49#include <sys/user.h>
50#undef _KERNEL
51#undef sigqueue
52#include <libutil.h>
53#endif
54#endif
Jun Nakajima432d2682010-08-31 16:41:25 +010055#else /* !CONFIG_USER_ONLY */
56#include "xen-mapcache.h"
pbrook53a59602006-03-25 19:31:22 +000057#endif
bellard54936002003-05-13 00:25:15 +000058
bellardfd6ce8f2003-05-14 19:00:11 +000059//#define DEBUG_TB_INVALIDATE
bellard66e85a22003-06-24 13:28:12 +000060//#define DEBUG_FLUSH
bellard9fa3e852004-01-04 18:06:42 +000061//#define DEBUG_TLB
pbrook67d3b952006-12-18 05:03:52 +000062//#define DEBUG_UNASSIGNED
bellardfd6ce8f2003-05-14 19:00:11 +000063
64/* make various TB consistency checks */
ths5fafdf22007-09-16 21:08:06 +000065//#define DEBUG_TB_CHECK
66//#define DEBUG_TLB_CHECK
bellardfd6ce8f2003-05-14 19:00:11 +000067
ths1196be32007-03-17 15:17:58 +000068//#define DEBUG_IOPORT
blueswir1db7b5422007-05-26 17:36:03 +000069//#define DEBUG_SUBPAGE
ths1196be32007-03-17 15:17:58 +000070
pbrook99773bd2006-04-16 15:14:59 +000071#if !defined(CONFIG_USER_ONLY)
72/* TB consistency checks only implemented for usermode emulation. */
73#undef DEBUG_TB_CHECK
74#endif
75
bellard9fa3e852004-01-04 18:06:42 +000076#define SMC_BITMAP_USE_THRESHOLD 10
77
blueswir1bdaf78e2008-10-04 07:24:27 +000078static TranslationBlock *tbs;
Stefan Weil24ab68a2010-07-19 18:23:17 +020079static int code_gen_max_blocks;
bellard9fa3e852004-01-04 18:06:42 +000080TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
blueswir1bdaf78e2008-10-04 07:24:27 +000081static int nb_tbs;
bellardeb51d102003-05-14 21:51:13 +000082/* any access to the tbs or the page table must use this lock */
Anthony Liguoric227f092009-10-01 16:12:16 -050083spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
bellardfd6ce8f2003-05-14 19:00:11 +000084
blueswir1141ac462008-07-26 15:05:57 +000085#if defined(__arm__) || defined(__sparc_v9__)
86/* The prologue must be reachable with a direct jump. ARM and Sparc64
87 have limited branch ranges (possibly also PPC) so place it in a
blueswir1d03d8602008-07-10 17:21:31 +000088 section close to code segment. */
89#define code_gen_section \
90 __attribute__((__section__(".gen_code"))) \
91 __attribute__((aligned (32)))
Stefan Weilf8e2af12009-06-18 23:04:48 +020092#elif defined(_WIN32)
93/* Maximum alignment for Win32 is 16. */
94#define code_gen_section \
95 __attribute__((aligned (16)))
blueswir1d03d8602008-07-10 17:21:31 +000096#else
97#define code_gen_section \
98 __attribute__((aligned (32)))
99#endif
100
101uint8_t code_gen_prologue[1024] code_gen_section;
blueswir1bdaf78e2008-10-04 07:24:27 +0000102static uint8_t *code_gen_buffer;
103static unsigned long code_gen_buffer_size;
bellard26a5f132008-05-28 12:30:31 +0000104/* threshold to flush the translated code buffer */
blueswir1bdaf78e2008-10-04 07:24:27 +0000105static unsigned long code_gen_buffer_max_size;
Stefan Weil24ab68a2010-07-19 18:23:17 +0200106static uint8_t *code_gen_ptr;
bellardfd6ce8f2003-05-14 19:00:11 +0000107
pbrooke2eef172008-06-08 01:09:01 +0000108#if !defined(CONFIG_USER_ONLY)
bellard9fa3e852004-01-04 18:06:42 +0000109int phys_ram_fd;
aliguori74576192008-10-06 14:02:03 +0000110static int in_migration;
pbrook94a6b542009-04-11 17:15:54 +0000111
Alex Williamsonf471a172010-06-11 11:11:42 -0600112RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
pbrooke2eef172008-06-08 01:09:01 +0000113#endif
bellard9fa3e852004-01-04 18:06:42 +0000114
bellard6a00d602005-11-21 23:25:50 +0000115CPUState *first_cpu;
116/* current CPU in the current thread. It is only valid inside
117 cpu_exec() */
ths5fafdf22007-09-16 21:08:06 +0000118CPUState *cpu_single_env;
pbrook2e70f6e2008-06-29 01:03:05 +0000119/* 0 = Do not count executed instructions.
thsbf20dc02008-06-30 17:22:19 +0000120 1 = Precise instruction counting.
pbrook2e70f6e2008-06-29 01:03:05 +0000121 2 = Adaptive rate instruction counting. */
122int use_icount = 0;
123/* Current instruction counter. While executing translated code this may
124 include some instructions that have not yet been executed. */
125int64_t qemu_icount;
bellard6a00d602005-11-21 23:25:50 +0000126
bellard54936002003-05-13 00:25:15 +0000127typedef struct PageDesc {
bellard92e873b2004-05-21 14:52:29 +0000128 /* list of TBs intersecting this ram page */
bellardfd6ce8f2003-05-14 19:00:11 +0000129 TranslationBlock *first_tb;
bellard9fa3e852004-01-04 18:06:42 +0000130 /* in order to optimize self modifying code, we count the number
131 of lookups we do to a given page to use a bitmap */
132 unsigned int code_write_count;
133 uint8_t *code_bitmap;
134#if defined(CONFIG_USER_ONLY)
135 unsigned long flags;
136#endif
bellard54936002003-05-13 00:25:15 +0000137} PageDesc;
138
Paul Brook41c1b1c2010-03-12 16:54:58 +0000139/* In system mode we want L1_MAP to be based on ram offsets,
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800140 while in user mode we want it to be based on virtual addresses. */
141#if !defined(CONFIG_USER_ONLY)
Paul Brook41c1b1c2010-03-12 16:54:58 +0000142#if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
143# define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
144#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800145# define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
Paul Brook41c1b1c2010-03-12 16:54:58 +0000146#endif
j_mayerbedb69e2007-04-05 20:08:21 +0000147#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800148# define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
j_mayerbedb69e2007-04-05 20:08:21 +0000149#endif
bellard54936002003-05-13 00:25:15 +0000150
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800151/* Size of the L2 (and L3, etc) page tables. */
152#define L2_BITS 10
bellard54936002003-05-13 00:25:15 +0000153#define L2_SIZE (1 << L2_BITS)
154
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800155/* The bits remaining after N lower levels of page tables. */
156#define P_L1_BITS_REM \
157 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158#define V_L1_BITS_REM \
159 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
160
161/* Size of the L1 page table. Avoid silly small sizes. */
162#if P_L1_BITS_REM < 4
163#define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
164#else
165#define P_L1_BITS P_L1_BITS_REM
166#endif
167
168#if V_L1_BITS_REM < 4
169#define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
170#else
171#define V_L1_BITS V_L1_BITS_REM
172#endif
173
174#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
175#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
176
177#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
178#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
179
bellard83fb7ad2004-07-05 21:25:26 +0000180unsigned long qemu_real_host_page_size;
181unsigned long qemu_host_page_bits;
182unsigned long qemu_host_page_size;
183unsigned long qemu_host_page_mask;
bellard54936002003-05-13 00:25:15 +0000184
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800185/* This is a multi-level map on the virtual address space.
186 The bottom level has pointers to PageDesc. */
187static void *l1_map[V_L1_SIZE];
bellard54936002003-05-13 00:25:15 +0000188
pbrooke2eef172008-06-08 01:09:01 +0000189#if !defined(CONFIG_USER_ONLY)
Paul Brook41c1b1c2010-03-12 16:54:58 +0000190typedef struct PhysPageDesc {
191 /* offset in host memory of the page + io_index in the low bits */
192 ram_addr_t phys_offset;
193 ram_addr_t region_offset;
194} PhysPageDesc;
195
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800196/* This is a multi-level map on the physical address space.
197 The bottom level has pointers to PhysPageDesc. */
198static void *l1_phys_map[P_L1_SIZE];
Paul Brook6d9a1302010-02-28 23:55:53 +0000199
pbrooke2eef172008-06-08 01:09:01 +0000200static void io_mem_init(void);
201
bellard33417e72003-08-10 21:47:01 +0000202/* io memory support */
bellard33417e72003-08-10 21:47:01 +0000203CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
204CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
bellarda4193c82004-06-03 14:01:43 +0000205void *io_mem_opaque[IO_MEM_NB_ENTRIES];
blueswir1511d2b12009-03-07 15:32:56 +0000206static char io_mem_used[IO_MEM_NB_ENTRIES];
pbrook6658ffb2007-03-16 23:58:11 +0000207static int io_mem_watch;
208#endif
bellard33417e72003-08-10 21:47:01 +0000209
bellard34865132003-10-05 14:28:56 +0000210/* log support */
Juha Riihimäki1e8b27c2009-12-03 15:56:02 +0200211#ifdef WIN32
212static const char *logfilename = "qemu.log";
213#else
blueswir1d9b630f2008-10-05 09:57:08 +0000214static const char *logfilename = "/tmp/qemu.log";
Juha Riihimäki1e8b27c2009-12-03 15:56:02 +0200215#endif
bellard34865132003-10-05 14:28:56 +0000216FILE *logfile;
217int loglevel;
pbrooke735b912007-06-30 13:53:24 +0000218static int log_append = 0;
bellard34865132003-10-05 14:28:56 +0000219
bellarde3db7222005-01-26 22:00:47 +0000220/* statistics */
Paul Brookb3755a92010-03-12 16:54:58 +0000221#if !defined(CONFIG_USER_ONLY)
bellarde3db7222005-01-26 22:00:47 +0000222static int tlb_flush_count;
Paul Brookb3755a92010-03-12 16:54:58 +0000223#endif
bellarde3db7222005-01-26 22:00:47 +0000224static int tb_flush_count;
225static int tb_phys_invalidate_count;
226
bellard7cb69ca2008-05-10 10:55:51 +0000227#ifdef _WIN32
228static void map_exec(void *addr, long size)
229{
230 DWORD old_protect;
231 VirtualProtect(addr, size,
232 PAGE_EXECUTE_READWRITE, &old_protect);
233
234}
235#else
236static void map_exec(void *addr, long size)
237{
bellard43694152008-05-29 09:35:57 +0000238 unsigned long start, end, page_size;
bellard7cb69ca2008-05-10 10:55:51 +0000239
bellard43694152008-05-29 09:35:57 +0000240 page_size = getpagesize();
bellard7cb69ca2008-05-10 10:55:51 +0000241 start = (unsigned long)addr;
bellard43694152008-05-29 09:35:57 +0000242 start &= ~(page_size - 1);
bellard7cb69ca2008-05-10 10:55:51 +0000243
244 end = (unsigned long)addr + size;
bellard43694152008-05-29 09:35:57 +0000245 end += page_size - 1;
246 end &= ~(page_size - 1);
bellard7cb69ca2008-05-10 10:55:51 +0000247
248 mprotect((void *)start, end - start,
249 PROT_READ | PROT_WRITE | PROT_EXEC);
250}
251#endif
252
bellardb346ff42003-06-15 20:05:50 +0000253static void page_init(void)
bellard54936002003-05-13 00:25:15 +0000254{
bellard83fb7ad2004-07-05 21:25:26 +0000255 /* NOTE: we can always suppose that qemu_host_page_size >=
bellard54936002003-05-13 00:25:15 +0000256 TARGET_PAGE_SIZE */
aliguoric2b48b62008-11-11 22:06:42 +0000257#ifdef _WIN32
258 {
259 SYSTEM_INFO system_info;
260
261 GetSystemInfo(&system_info);
262 qemu_real_host_page_size = system_info.dwPageSize;
263 }
264#else
265 qemu_real_host_page_size = getpagesize();
266#endif
bellard83fb7ad2004-07-05 21:25:26 +0000267 if (qemu_host_page_size == 0)
268 qemu_host_page_size = qemu_real_host_page_size;
269 if (qemu_host_page_size < TARGET_PAGE_SIZE)
270 qemu_host_page_size = TARGET_PAGE_SIZE;
271 qemu_host_page_bits = 0;
272 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
273 qemu_host_page_bits++;
274 qemu_host_page_mask = ~(qemu_host_page_size - 1);
balrog50a95692007-12-12 01:16:23 +0000275
Paul Brook2e9a5712010-05-05 16:32:59 +0100276#if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
balrog50a95692007-12-12 01:16:23 +0000277 {
Juergen Lockf01576f2010-03-25 22:32:16 +0100278#ifdef HAVE_KINFO_GETVMMAP
279 struct kinfo_vmentry *freep;
280 int i, cnt;
281
282 freep = kinfo_getvmmap(getpid(), &cnt);
283 if (freep) {
284 mmap_lock();
285 for (i = 0; i < cnt; i++) {
286 unsigned long startaddr, endaddr;
287
288 startaddr = freep[i].kve_start;
289 endaddr = freep[i].kve_end;
290 if (h2g_valid(startaddr)) {
291 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
292
293 if (h2g_valid(endaddr)) {
294 endaddr = h2g(endaddr);
Aurelien Jarnofd436902010-04-10 17:20:36 +0200295 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
Juergen Lockf01576f2010-03-25 22:32:16 +0100296 } else {
297#if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
298 endaddr = ~0ul;
Aurelien Jarnofd436902010-04-10 17:20:36 +0200299 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
Juergen Lockf01576f2010-03-25 22:32:16 +0100300#endif
301 }
302 }
303 }
304 free(freep);
305 mmap_unlock();
306 }
307#else
balrog50a95692007-12-12 01:16:23 +0000308 FILE *f;
balrog50a95692007-12-12 01:16:23 +0000309
pbrook07765902008-05-31 16:33:53 +0000310 last_brk = (unsigned long)sbrk(0);
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800311
Aurelien Jarnofd436902010-04-10 17:20:36 +0200312 f = fopen("/compat/linux/proc/self/maps", "r");
balrog50a95692007-12-12 01:16:23 +0000313 if (f) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800314 mmap_lock();
315
balrog50a95692007-12-12 01:16:23 +0000316 do {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800317 unsigned long startaddr, endaddr;
318 int n;
319
320 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
321
322 if (n == 2 && h2g_valid(startaddr)) {
323 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
324
325 if (h2g_valid(endaddr)) {
326 endaddr = h2g(endaddr);
327 } else {
328 endaddr = ~0ul;
329 }
330 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
balrog50a95692007-12-12 01:16:23 +0000331 }
332 } while (!feof(f));
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800333
balrog50a95692007-12-12 01:16:23 +0000334 fclose(f);
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800335 mmap_unlock();
balrog50a95692007-12-12 01:16:23 +0000336 }
Juergen Lockf01576f2010-03-25 22:32:16 +0100337#endif
balrog50a95692007-12-12 01:16:23 +0000338 }
339#endif
bellard54936002003-05-13 00:25:15 +0000340}
341
Paul Brook41c1b1c2010-03-12 16:54:58 +0000342static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
bellard54936002003-05-13 00:25:15 +0000343{
Paul Brook41c1b1c2010-03-12 16:54:58 +0000344 PageDesc *pd;
345 void **lp;
346 int i;
347
pbrook17e23772008-06-09 13:47:45 +0000348#if defined(CONFIG_USER_ONLY)
Paul Brook2e9a5712010-05-05 16:32:59 +0100349 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800350# define ALLOC(P, SIZE) \
351 do { \
352 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
353 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800354 } while (0)
pbrook17e23772008-06-09 13:47:45 +0000355#else
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800356# define ALLOC(P, SIZE) \
357 do { P = qemu_mallocz(SIZE); } while (0)
pbrook17e23772008-06-09 13:47:45 +0000358#endif
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800359
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800360 /* Level 1. Always allocated. */
361 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
362
363 /* Level 2..N-1. */
364 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
365 void **p = *lp;
366
367 if (p == NULL) {
368 if (!alloc) {
369 return NULL;
370 }
371 ALLOC(p, sizeof(void *) * L2_SIZE);
372 *lp = p;
373 }
374
375 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
bellard54936002003-05-13 00:25:15 +0000376 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800377
378 pd = *lp;
379 if (pd == NULL) {
380 if (!alloc) {
381 return NULL;
382 }
383 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
384 *lp = pd;
385 }
386
387#undef ALLOC
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800388
389 return pd + (index & (L2_SIZE - 1));
bellard54936002003-05-13 00:25:15 +0000390}
391
Paul Brook41c1b1c2010-03-12 16:54:58 +0000392static inline PageDesc *page_find(tb_page_addr_t index)
bellard54936002003-05-13 00:25:15 +0000393{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800394 return page_find_alloc(index, 0);
bellard54936002003-05-13 00:25:15 +0000395}
396
Paul Brook6d9a1302010-02-28 23:55:53 +0000397#if !defined(CONFIG_USER_ONLY)
Anthony Liguoric227f092009-10-01 16:12:16 -0500398static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
bellard92e873b2004-05-21 14:52:29 +0000399{
pbrooke3f4e2a2006-04-08 20:02:06 +0000400 PhysPageDesc *pd;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800401 void **lp;
402 int i;
bellard92e873b2004-05-21 14:52:29 +0000403
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800404 /* Level 1. Always allocated. */
405 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
bellard108c49b2005-07-24 12:55:09 +0000406
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800407 /* Level 2..N-1. */
408 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
409 void **p = *lp;
410 if (p == NULL) {
411 if (!alloc) {
412 return NULL;
413 }
414 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
415 }
416 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
bellard108c49b2005-07-24 12:55:09 +0000417 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800418
pbrooke3f4e2a2006-04-08 20:02:06 +0000419 pd = *lp;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800420 if (pd == NULL) {
pbrooke3f4e2a2006-04-08 20:02:06 +0000421 int i;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800422
423 if (!alloc) {
bellard108c49b2005-07-24 12:55:09 +0000424 return NULL;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800425 }
426
427 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
428
pbrook67c4d232009-02-23 13:16:07 +0000429 for (i = 0; i < L2_SIZE; i++) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800430 pd[i].phys_offset = IO_MEM_UNASSIGNED;
431 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
pbrook67c4d232009-02-23 13:16:07 +0000432 }
bellard92e873b2004-05-21 14:52:29 +0000433 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800434
435 return pd + (index & (L2_SIZE - 1));
bellard92e873b2004-05-21 14:52:29 +0000436}
437
Anthony Liguoric227f092009-10-01 16:12:16 -0500438static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
bellard92e873b2004-05-21 14:52:29 +0000439{
bellard108c49b2005-07-24 12:55:09 +0000440 return phys_page_find_alloc(index, 0);
bellard92e873b2004-05-21 14:52:29 +0000441}
442
Anthony Liguoric227f092009-10-01 16:12:16 -0500443static void tlb_protect_code(ram_addr_t ram_addr);
444static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
bellard3a7d9292005-08-21 09:26:42 +0000445 target_ulong vaddr);
pbrookc8a706f2008-06-02 16:16:42 +0000446#define mmap_lock() do { } while(0)
447#define mmap_unlock() do { } while(0)
bellard9fa3e852004-01-04 18:06:42 +0000448#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000449
bellard43694152008-05-29 09:35:57 +0000450#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
451
452#if defined(CONFIG_USER_ONLY)
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100453/* Currently it is not recommended to allocate big chunks of data in
bellard43694152008-05-29 09:35:57 +0000454 user mode. It will change when a dedicated libc will be used */
455#define USE_STATIC_CODE_GEN_BUFFER
456#endif
457
458#ifdef USE_STATIC_CODE_GEN_BUFFER
Aurelien Jarnoebf50fb2010-03-29 02:12:51 +0200459static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
460 __attribute__((aligned (CODE_GEN_ALIGN)));
bellard43694152008-05-29 09:35:57 +0000461#endif
462
blueswir18fcd3692008-08-17 20:26:25 +0000463static void code_gen_alloc(unsigned long tb_size)
bellard26a5f132008-05-28 12:30:31 +0000464{
bellard43694152008-05-29 09:35:57 +0000465#ifdef USE_STATIC_CODE_GEN_BUFFER
466 code_gen_buffer = static_code_gen_buffer;
467 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
468 map_exec(code_gen_buffer, code_gen_buffer_size);
469#else
bellard26a5f132008-05-28 12:30:31 +0000470 code_gen_buffer_size = tb_size;
471 if (code_gen_buffer_size == 0) {
bellard43694152008-05-29 09:35:57 +0000472#if defined(CONFIG_USER_ONLY)
473 /* in user mode, phys_ram_size is not meaningful */
474 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
475#else
Stuart Bradyccbb4d42009-05-03 12:15:06 +0100476 /* XXX: needs adjustments */
pbrook94a6b542009-04-11 17:15:54 +0000477 code_gen_buffer_size = (unsigned long)(ram_size / 4);
bellard43694152008-05-29 09:35:57 +0000478#endif
bellard26a5f132008-05-28 12:30:31 +0000479 }
480 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
481 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
482 /* The code gen buffer location may have constraints depending on
483 the host cpu and OS */
484#if defined(__linux__)
485 {
486 int flags;
blueswir1141ac462008-07-26 15:05:57 +0000487 void *start = NULL;
488
bellard26a5f132008-05-28 12:30:31 +0000489 flags = MAP_PRIVATE | MAP_ANONYMOUS;
490#if defined(__x86_64__)
491 flags |= MAP_32BIT;
492 /* Cannot map more than that */
493 if (code_gen_buffer_size > (800 * 1024 * 1024))
494 code_gen_buffer_size = (800 * 1024 * 1024);
blueswir1141ac462008-07-26 15:05:57 +0000495#elif defined(__sparc_v9__)
496 // Map the buffer below 2G, so we can use direct calls and branches
497 flags |= MAP_FIXED;
498 start = (void *) 0x60000000UL;
499 if (code_gen_buffer_size > (512 * 1024 * 1024))
500 code_gen_buffer_size = (512 * 1024 * 1024);
balrog1cb06612008-12-01 02:10:17 +0000501#elif defined(__arm__)
balrog63d41242008-12-01 02:19:41 +0000502 /* Map the buffer below 32M, so we can use direct calls and branches */
balrog1cb06612008-12-01 02:10:17 +0000503 flags |= MAP_FIXED;
504 start = (void *) 0x01000000UL;
505 if (code_gen_buffer_size > 16 * 1024 * 1024)
506 code_gen_buffer_size = 16 * 1024 * 1024;
Richard Hendersoneba0b892010-06-04 12:14:14 -0700507#elif defined(__s390x__)
508 /* Map the buffer so that we can use direct calls and branches. */
509 /* We have a +- 4GB range on the branches; leave some slop. */
510 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
511 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
512 }
513 start = (void *)0x90000000UL;
bellard26a5f132008-05-28 12:30:31 +0000514#endif
blueswir1141ac462008-07-26 15:05:57 +0000515 code_gen_buffer = mmap(start, code_gen_buffer_size,
516 PROT_WRITE | PROT_READ | PROT_EXEC,
bellard26a5f132008-05-28 12:30:31 +0000517 flags, -1, 0);
518 if (code_gen_buffer == MAP_FAILED) {
519 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
520 exit(1);
521 }
522 }
Bradcbb608a2010-12-20 21:25:40 -0500523#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
524 || defined(__DragonFly__) || defined(__OpenBSD__)
aliguori06e67a82008-09-27 15:32:41 +0000525 {
526 int flags;
527 void *addr = NULL;
528 flags = MAP_PRIVATE | MAP_ANONYMOUS;
529#if defined(__x86_64__)
530 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
531 * 0x40000000 is free */
532 flags |= MAP_FIXED;
533 addr = (void *)0x40000000;
534 /* Cannot map more than that */
535 if (code_gen_buffer_size > (800 * 1024 * 1024))
536 code_gen_buffer_size = (800 * 1024 * 1024);
Blue Swirl4cd31ad2011-01-16 08:32:27 +0000537#elif defined(__sparc_v9__)
538 // Map the buffer below 2G, so we can use direct calls and branches
539 flags |= MAP_FIXED;
540 addr = (void *) 0x60000000UL;
541 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
542 code_gen_buffer_size = (512 * 1024 * 1024);
543 }
aliguori06e67a82008-09-27 15:32:41 +0000544#endif
545 code_gen_buffer = mmap(addr, code_gen_buffer_size,
546 PROT_WRITE | PROT_READ | PROT_EXEC,
547 flags, -1, 0);
548 if (code_gen_buffer == MAP_FAILED) {
549 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
550 exit(1);
551 }
552 }
bellard26a5f132008-05-28 12:30:31 +0000553#else
554 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
bellard26a5f132008-05-28 12:30:31 +0000555 map_exec(code_gen_buffer, code_gen_buffer_size);
556#endif
bellard43694152008-05-29 09:35:57 +0000557#endif /* !USE_STATIC_CODE_GEN_BUFFER */
bellard26a5f132008-05-28 12:30:31 +0000558 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
559 code_gen_buffer_max_size = code_gen_buffer_size -
Aurelien Jarno239fda32010-06-03 19:29:31 +0200560 (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
bellard26a5f132008-05-28 12:30:31 +0000561 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
562 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
563}
564
565/* Must be called before using the QEMU cpus. 'tb_size' is the size
566 (in bytes) allocated to the translation buffer. Zero means default
567 size. */
568void cpu_exec_init_all(unsigned long tb_size)
569{
bellard26a5f132008-05-28 12:30:31 +0000570 cpu_gen_init();
571 code_gen_alloc(tb_size);
572 code_gen_ptr = code_gen_buffer;
bellard43694152008-05-29 09:35:57 +0000573 page_init();
pbrooke2eef172008-06-08 01:09:01 +0000574#if !defined(CONFIG_USER_ONLY)
bellard26a5f132008-05-28 12:30:31 +0000575 io_mem_init();
pbrooke2eef172008-06-08 01:09:01 +0000576#endif
Richard Henderson9002ec72010-05-06 08:50:41 -0700577#if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
578 /* There's no guest base to take into account, so go ahead and
579 initialize the prologue now. */
580 tcg_prologue_init(&tcg_ctx);
581#endif
bellard26a5f132008-05-28 12:30:31 +0000582}
583
pbrook9656f322008-07-01 20:01:19 +0000584#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
585
Juan Quintelae59fb372009-09-29 22:48:21 +0200586static int cpu_common_post_load(void *opaque, int version_id)
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200587{
588 CPUState *env = opaque;
589
aurel323098dba2009-03-07 21:28:24 +0000590 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
591 version_id is increased. */
592 env->interrupt_request &= ~0x01;
pbrook9656f322008-07-01 20:01:19 +0000593 tlb_flush(env, 1);
594
595 return 0;
596}
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200597
598static const VMStateDescription vmstate_cpu_common = {
599 .name = "cpu_common",
600 .version_id = 1,
601 .minimum_version_id = 1,
602 .minimum_version_id_old = 1,
Juan Quintelae7f4eff2009-09-10 03:04:33 +0200603 .post_load = cpu_common_post_load,
604 .fields = (VMStateField []) {
605 VMSTATE_UINT32(halted, CPUState),
606 VMSTATE_UINT32(interrupt_request, CPUState),
607 VMSTATE_END_OF_LIST()
608 }
609};
pbrook9656f322008-07-01 20:01:19 +0000610#endif
611
Glauber Costa950f1472009-06-09 12:15:18 -0400612CPUState *qemu_get_cpu(int cpu)
613{
614 CPUState *env = first_cpu;
615
616 while (env) {
617 if (env->cpu_index == cpu)
618 break;
619 env = env->next_cpu;
620 }
621
622 return env;
623}
624
bellard6a00d602005-11-21 23:25:50 +0000625void cpu_exec_init(CPUState *env)
bellardfd6ce8f2003-05-14 19:00:11 +0000626{
bellard6a00d602005-11-21 23:25:50 +0000627 CPUState **penv;
628 int cpu_index;
629
pbrookc2764712009-03-07 15:24:59 +0000630#if defined(CONFIG_USER_ONLY)
631 cpu_list_lock();
632#endif
bellard6a00d602005-11-21 23:25:50 +0000633 env->next_cpu = NULL;
634 penv = &first_cpu;
635 cpu_index = 0;
636 while (*penv != NULL) {
Nathan Froyd1e9fa732009-06-03 11:33:08 -0700637 penv = &(*penv)->next_cpu;
bellard6a00d602005-11-21 23:25:50 +0000638 cpu_index++;
639 }
640 env->cpu_index = cpu_index;
aliguori268a3622009-04-21 22:30:27 +0000641 env->numa_node = 0;
Blue Swirl72cf2d42009-09-12 07:36:22 +0000642 QTAILQ_INIT(&env->breakpoints);
643 QTAILQ_INIT(&env->watchpoints);
Jan Kiszkadc7a09c2011-03-15 12:26:31 +0100644#ifndef CONFIG_USER_ONLY
645 env->thread_id = qemu_get_thread_id();
646#endif
bellard6a00d602005-11-21 23:25:50 +0000647 *penv = env;
pbrookc2764712009-03-07 15:24:59 +0000648#if defined(CONFIG_USER_ONLY)
649 cpu_list_unlock();
650#endif
pbrookb3c77242008-06-30 16:31:04 +0000651#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
Alex Williamson0be71e32010-06-25 11:09:07 -0600652 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
653 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
pbrookb3c77242008-06-30 16:31:04 +0000654 cpu_save, cpu_load, env);
655#endif
bellardfd6ce8f2003-05-14 19:00:11 +0000656}
657
Tristan Gingoldd1a1eb72011-02-10 10:04:57 +0100658/* Allocate a new translation block. Flush the translation buffer if
659 too many translation blocks or too much generated code. */
660static TranslationBlock *tb_alloc(target_ulong pc)
661{
662 TranslationBlock *tb;
663
664 if (nb_tbs >= code_gen_max_blocks ||
665 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
666 return NULL;
667 tb = &tbs[nb_tbs++];
668 tb->pc = pc;
669 tb->cflags = 0;
670 return tb;
671}
672
673void tb_free(TranslationBlock *tb)
674{
675 /* In practice this is mostly used for single use temporary TB
676 Ignore the hard cases and just back up if this TB happens to
677 be the last one generated. */
678 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
679 code_gen_ptr = tb->tc_ptr;
680 nb_tbs--;
681 }
682}
683
bellard9fa3e852004-01-04 18:06:42 +0000684static inline void invalidate_page_bitmap(PageDesc *p)
685{
686 if (p->code_bitmap) {
bellard59817cc2004-02-16 22:01:13 +0000687 qemu_free(p->code_bitmap);
bellard9fa3e852004-01-04 18:06:42 +0000688 p->code_bitmap = NULL;
689 }
690 p->code_write_count = 0;
691}
692
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800693/* Set to NULL all the 'first_tb' fields in all PageDescs. */
694
695static void page_flush_tb_1 (int level, void **lp)
696{
697 int i;
698
699 if (*lp == NULL) {
700 return;
701 }
702 if (level == 0) {
703 PageDesc *pd = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +0000704 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800705 pd[i].first_tb = NULL;
706 invalidate_page_bitmap(pd + i);
707 }
708 } else {
709 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +0000710 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800711 page_flush_tb_1 (level - 1, pp + i);
712 }
713 }
714}
715
bellardfd6ce8f2003-05-14 19:00:11 +0000716static void page_flush_tb(void)
717{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -0800718 int i;
719 for (i = 0; i < V_L1_SIZE; i++) {
720 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
bellardfd6ce8f2003-05-14 19:00:11 +0000721 }
722}
723
724/* flush all the translation blocks */
bellardd4e81642003-05-25 16:46:15 +0000725/* XXX: tb_flush is currently not thread safe */
bellard6a00d602005-11-21 23:25:50 +0000726void tb_flush(CPUState *env1)
bellardfd6ce8f2003-05-14 19:00:11 +0000727{
bellard6a00d602005-11-21 23:25:50 +0000728 CPUState *env;
bellard01243112004-01-04 15:48:17 +0000729#if defined(DEBUG_FLUSH)
blueswir1ab3d1722007-11-04 07:31:40 +0000730 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
731 (unsigned long)(code_gen_ptr - code_gen_buffer),
732 nb_tbs, nb_tbs > 0 ?
733 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
bellardfd6ce8f2003-05-14 19:00:11 +0000734#endif
bellard26a5f132008-05-28 12:30:31 +0000735 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
pbrooka208e542008-03-31 17:07:36 +0000736 cpu_abort(env1, "Internal error: code buffer overflow\n");
737
bellardfd6ce8f2003-05-14 19:00:11 +0000738 nb_tbs = 0;
ths3b46e622007-09-17 08:09:54 +0000739
bellard6a00d602005-11-21 23:25:50 +0000740 for(env = first_cpu; env != NULL; env = env->next_cpu) {
741 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
742 }
bellard9fa3e852004-01-04 18:06:42 +0000743
bellard8a8a6082004-10-03 13:36:49 +0000744 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
bellardfd6ce8f2003-05-14 19:00:11 +0000745 page_flush_tb();
bellard9fa3e852004-01-04 18:06:42 +0000746
bellardfd6ce8f2003-05-14 19:00:11 +0000747 code_gen_ptr = code_gen_buffer;
bellardd4e81642003-05-25 16:46:15 +0000748 /* XXX: flush processor icache at this point if cache flush is
749 expensive */
bellarde3db7222005-01-26 22:00:47 +0000750 tb_flush_count++;
bellardfd6ce8f2003-05-14 19:00:11 +0000751}
752
753#ifdef DEBUG_TB_CHECK
754
j_mayerbc98a7e2007-04-04 07:55:12 +0000755static void tb_invalidate_check(target_ulong address)
bellardfd6ce8f2003-05-14 19:00:11 +0000756{
757 TranslationBlock *tb;
758 int i;
759 address &= TARGET_PAGE_MASK;
pbrook99773bd2006-04-16 15:14:59 +0000760 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
761 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellardfd6ce8f2003-05-14 19:00:11 +0000762 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
763 address >= tb->pc + tb->size)) {
Blue Swirl0bf9e312009-07-20 17:19:25 +0000764 printf("ERROR invalidate: address=" TARGET_FMT_lx
765 " PC=%08lx size=%04x\n",
pbrook99773bd2006-04-16 15:14:59 +0000766 address, (long)tb->pc, tb->size);
bellardfd6ce8f2003-05-14 19:00:11 +0000767 }
768 }
769 }
770}
771
772/* verify that all the pages have correct rights for code */
773static void tb_page_check(void)
774{
775 TranslationBlock *tb;
776 int i, flags1, flags2;
ths3b46e622007-09-17 08:09:54 +0000777
pbrook99773bd2006-04-16 15:14:59 +0000778 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
779 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
bellardfd6ce8f2003-05-14 19:00:11 +0000780 flags1 = page_get_flags(tb->pc);
781 flags2 = page_get_flags(tb->pc + tb->size - 1);
782 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
783 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
pbrook99773bd2006-04-16 15:14:59 +0000784 (long)tb->pc, tb->size, flags1, flags2);
bellardfd6ce8f2003-05-14 19:00:11 +0000785 }
786 }
787 }
788}
789
790#endif
791
792/* invalidate one TB */
793static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
794 int next_offset)
795{
796 TranslationBlock *tb1;
797 for(;;) {
798 tb1 = *ptb;
799 if (tb1 == tb) {
800 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
801 break;
802 }
803 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
804 }
805}
806
bellard9fa3e852004-01-04 18:06:42 +0000807static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
808{
809 TranslationBlock *tb1;
810 unsigned int n1;
811
812 for(;;) {
813 tb1 = *ptb;
814 n1 = (long)tb1 & 3;
815 tb1 = (TranslationBlock *)((long)tb1 & ~3);
816 if (tb1 == tb) {
817 *ptb = tb1->page_next[n1];
818 break;
819 }
820 ptb = &tb1->page_next[n1];
821 }
822}
823
bellardd4e81642003-05-25 16:46:15 +0000824static inline void tb_jmp_remove(TranslationBlock *tb, int n)
825{
826 TranslationBlock *tb1, **ptb;
827 unsigned int n1;
828
829 ptb = &tb->jmp_next[n];
830 tb1 = *ptb;
831 if (tb1) {
832 /* find tb(n) in circular list */
833 for(;;) {
834 tb1 = *ptb;
835 n1 = (long)tb1 & 3;
836 tb1 = (TranslationBlock *)((long)tb1 & ~3);
837 if (n1 == n && tb1 == tb)
838 break;
839 if (n1 == 2) {
840 ptb = &tb1->jmp_first;
841 } else {
842 ptb = &tb1->jmp_next[n1];
843 }
844 }
845 /* now we can suppress tb(n) from the list */
846 *ptb = tb->jmp_next[n];
847
848 tb->jmp_next[n] = NULL;
849 }
850}
851
852/* reset the jump entry 'n' of a TB so that it is not chained to
853 another TB */
854static inline void tb_reset_jump(TranslationBlock *tb, int n)
855{
856 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
857}
858
Paul Brook41c1b1c2010-03-12 16:54:58 +0000859void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
bellardfd6ce8f2003-05-14 19:00:11 +0000860{
bellard6a00d602005-11-21 23:25:50 +0000861 CPUState *env;
bellardfd6ce8f2003-05-14 19:00:11 +0000862 PageDesc *p;
bellard8a40a182005-11-20 10:35:40 +0000863 unsigned int h, n1;
Paul Brook41c1b1c2010-03-12 16:54:58 +0000864 tb_page_addr_t phys_pc;
bellard8a40a182005-11-20 10:35:40 +0000865 TranslationBlock *tb1, *tb2;
ths3b46e622007-09-17 08:09:54 +0000866
bellard9fa3e852004-01-04 18:06:42 +0000867 /* remove the TB from the hash list */
868 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
869 h = tb_phys_hash_func(phys_pc);
ths5fafdf22007-09-16 21:08:06 +0000870 tb_remove(&tb_phys_hash[h], tb,
bellard9fa3e852004-01-04 18:06:42 +0000871 offsetof(TranslationBlock, phys_hash_next));
bellardfd6ce8f2003-05-14 19:00:11 +0000872
bellard9fa3e852004-01-04 18:06:42 +0000873 /* remove the TB from the page list */
874 if (tb->page_addr[0] != page_addr) {
875 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
876 tb_page_remove(&p->first_tb, tb);
877 invalidate_page_bitmap(p);
878 }
879 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
880 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
881 tb_page_remove(&p->first_tb, tb);
882 invalidate_page_bitmap(p);
883 }
884
bellard8a40a182005-11-20 10:35:40 +0000885 tb_invalidated_flag = 1;
886
887 /* remove the TB from the hash list */
888 h = tb_jmp_cache_hash_func(tb->pc);
bellard6a00d602005-11-21 23:25:50 +0000889 for(env = first_cpu; env != NULL; env = env->next_cpu) {
890 if (env->tb_jmp_cache[h] == tb)
891 env->tb_jmp_cache[h] = NULL;
892 }
bellard8a40a182005-11-20 10:35:40 +0000893
894 /* suppress this TB from the two jump lists */
895 tb_jmp_remove(tb, 0);
896 tb_jmp_remove(tb, 1);
897
898 /* suppress any remaining jumps to this TB */
899 tb1 = tb->jmp_first;
900 for(;;) {
901 n1 = (long)tb1 & 3;
902 if (n1 == 2)
903 break;
904 tb1 = (TranslationBlock *)((long)tb1 & ~3);
905 tb2 = tb1->jmp_next[n1];
906 tb_reset_jump(tb1, n1);
907 tb1->jmp_next[n1] = NULL;
908 tb1 = tb2;
909 }
910 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
911
bellarde3db7222005-01-26 22:00:47 +0000912 tb_phys_invalidate_count++;
bellard9fa3e852004-01-04 18:06:42 +0000913}
914
915static inline void set_bits(uint8_t *tab, int start, int len)
916{
917 int end, mask, end1;
918
919 end = start + len;
920 tab += start >> 3;
921 mask = 0xff << (start & 7);
922 if ((start & ~7) == (end & ~7)) {
923 if (start < end) {
924 mask &= ~(0xff << (end & 7));
925 *tab |= mask;
926 }
927 } else {
928 *tab++ |= mask;
929 start = (start + 8) & ~7;
930 end1 = end & ~7;
931 while (start < end1) {
932 *tab++ = 0xff;
933 start += 8;
934 }
935 if (start < end) {
936 mask = ~(0xff << (end & 7));
937 *tab |= mask;
938 }
939 }
940}
941
942static void build_page_bitmap(PageDesc *p)
943{
944 int n, tb_start, tb_end;
945 TranslationBlock *tb;
ths3b46e622007-09-17 08:09:54 +0000946
pbrookb2a70812008-06-09 13:57:23 +0000947 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
bellard9fa3e852004-01-04 18:06:42 +0000948
949 tb = p->first_tb;
950 while (tb != NULL) {
951 n = (long)tb & 3;
952 tb = (TranslationBlock *)((long)tb & ~3);
953 /* NOTE: this is subtle as a TB may span two physical pages */
954 if (n == 0) {
955 /* NOTE: tb_end may be after the end of the page, but
956 it is not a problem */
957 tb_start = tb->pc & ~TARGET_PAGE_MASK;
958 tb_end = tb_start + tb->size;
959 if (tb_end > TARGET_PAGE_SIZE)
960 tb_end = TARGET_PAGE_SIZE;
961 } else {
962 tb_start = 0;
963 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
964 }
965 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
966 tb = tb->page_next[n];
967 }
968}
969
pbrook2e70f6e2008-06-29 01:03:05 +0000970TranslationBlock *tb_gen_code(CPUState *env,
971 target_ulong pc, target_ulong cs_base,
972 int flags, int cflags)
bellardd720b932004-04-25 17:57:43 +0000973{
974 TranslationBlock *tb;
975 uint8_t *tc_ptr;
Paul Brook41c1b1c2010-03-12 16:54:58 +0000976 tb_page_addr_t phys_pc, phys_page2;
977 target_ulong virt_page2;
bellardd720b932004-04-25 17:57:43 +0000978 int code_gen_size;
979
Paul Brook41c1b1c2010-03-12 16:54:58 +0000980 phys_pc = get_page_addr_code(env, pc);
bellardc27004e2005-01-03 23:35:10 +0000981 tb = tb_alloc(pc);
bellardd720b932004-04-25 17:57:43 +0000982 if (!tb) {
983 /* flush must be done */
984 tb_flush(env);
985 /* cannot fail at this point */
bellardc27004e2005-01-03 23:35:10 +0000986 tb = tb_alloc(pc);
pbrook2e70f6e2008-06-29 01:03:05 +0000987 /* Don't forget to invalidate previous TB info. */
988 tb_invalidated_flag = 1;
bellardd720b932004-04-25 17:57:43 +0000989 }
990 tc_ptr = code_gen_ptr;
991 tb->tc_ptr = tc_ptr;
992 tb->cs_base = cs_base;
993 tb->flags = flags;
994 tb->cflags = cflags;
blueswir1d07bde82007-12-11 19:35:45 +0000995 cpu_gen_code(env, tb, &code_gen_size);
bellardd720b932004-04-25 17:57:43 +0000996 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
ths3b46e622007-09-17 08:09:54 +0000997
bellardd720b932004-04-25 17:57:43 +0000998 /* check next page if needed */
bellardc27004e2005-01-03 23:35:10 +0000999 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
bellardd720b932004-04-25 17:57:43 +00001000 phys_page2 = -1;
bellardc27004e2005-01-03 23:35:10 +00001001 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
Paul Brook41c1b1c2010-03-12 16:54:58 +00001002 phys_page2 = get_page_addr_code(env, virt_page2);
bellardd720b932004-04-25 17:57:43 +00001003 }
Paul Brook41c1b1c2010-03-12 16:54:58 +00001004 tb_link_page(tb, phys_pc, phys_page2);
pbrook2e70f6e2008-06-29 01:03:05 +00001005 return tb;
bellardd720b932004-04-25 17:57:43 +00001006}
ths3b46e622007-09-17 08:09:54 +00001007
bellard9fa3e852004-01-04 18:06:42 +00001008/* invalidate all TBs which intersect with the target physical page
1009 starting in range [start;end[. NOTE: start and end must refer to
bellardd720b932004-04-25 17:57:43 +00001010 the same physical page. 'is_cpu_write_access' should be true if called
1011 from a real cpu write access: the virtual CPU will exit the current
1012 TB if code is modified inside this TB. */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001013void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
bellardd720b932004-04-25 17:57:43 +00001014 int is_cpu_write_access)
bellard9fa3e852004-01-04 18:06:42 +00001015{
aliguori6b917542008-11-18 19:46:41 +00001016 TranslationBlock *tb, *tb_next, *saved_tb;
bellardd720b932004-04-25 17:57:43 +00001017 CPUState *env = cpu_single_env;
Paul Brook41c1b1c2010-03-12 16:54:58 +00001018 tb_page_addr_t tb_start, tb_end;
aliguori6b917542008-11-18 19:46:41 +00001019 PageDesc *p;
1020 int n;
1021#ifdef TARGET_HAS_PRECISE_SMC
1022 int current_tb_not_found = is_cpu_write_access;
1023 TranslationBlock *current_tb = NULL;
1024 int current_tb_modified = 0;
1025 target_ulong current_pc = 0;
1026 target_ulong current_cs_base = 0;
1027 int current_flags = 0;
1028#endif /* TARGET_HAS_PRECISE_SMC */
bellard9fa3e852004-01-04 18:06:42 +00001029
1030 p = page_find(start >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001031 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00001032 return;
ths5fafdf22007-09-16 21:08:06 +00001033 if (!p->code_bitmap &&
bellardd720b932004-04-25 17:57:43 +00001034 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1035 is_cpu_write_access) {
bellard9fa3e852004-01-04 18:06:42 +00001036 /* build code bitmap */
1037 build_page_bitmap(p);
1038 }
1039
1040 /* we remove all the TBs in the range [start, end[ */
1041 /* XXX: see if in some cases it could be faster to invalidate all the code */
1042 tb = p->first_tb;
1043 while (tb != NULL) {
1044 n = (long)tb & 3;
1045 tb = (TranslationBlock *)((long)tb & ~3);
1046 tb_next = tb->page_next[n];
1047 /* NOTE: this is subtle as a TB may span two physical pages */
1048 if (n == 0) {
1049 /* NOTE: tb_end may be after the end of the page, but
1050 it is not a problem */
1051 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1052 tb_end = tb_start + tb->size;
1053 } else {
1054 tb_start = tb->page_addr[1];
1055 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1056 }
1057 if (!(tb_end <= start || tb_start >= end)) {
bellardd720b932004-04-25 17:57:43 +00001058#ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb_not_found) {
1060 current_tb_not_found = 0;
1061 current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001062 if (env->mem_io_pc) {
bellardd720b932004-04-25 17:57:43 +00001063 /* now we have a real cpu fault */
pbrook2e70f6e2008-06-29 01:03:05 +00001064 current_tb = tb_find_pc(env->mem_io_pc);
bellardd720b932004-04-25 17:57:43 +00001065 }
1066 }
1067 if (current_tb == tb &&
pbrook2e70f6e2008-06-29 01:03:05 +00001068 (current_tb->cflags & CF_COUNT_MASK) != 1) {
bellardd720b932004-04-25 17:57:43 +00001069 /* If we are modifying the current TB, we must stop
1070 its execution. We could be more precise by checking
1071 that the modification is after the current PC, but it
1072 would require a specialized function to partially
1073 restore the CPU state */
ths3b46e622007-09-17 08:09:54 +00001074
bellardd720b932004-04-25 17:57:43 +00001075 current_tb_modified = 1;
Stefan Weil618ba8e2011-04-18 06:39:53 +00001076 cpu_restore_state(current_tb, env, env->mem_io_pc);
aliguori6b917542008-11-18 19:46:41 +00001077 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1078 &current_flags);
bellardd720b932004-04-25 17:57:43 +00001079 }
1080#endif /* TARGET_HAS_PRECISE_SMC */
bellard6f5a9f72005-11-26 20:12:28 +00001081 /* we need to do that to handle the case where a signal
1082 occurs while doing tb_phys_invalidate() */
1083 saved_tb = NULL;
1084 if (env) {
1085 saved_tb = env->current_tb;
1086 env->current_tb = NULL;
1087 }
bellard9fa3e852004-01-04 18:06:42 +00001088 tb_phys_invalidate(tb, -1);
bellard6f5a9f72005-11-26 20:12:28 +00001089 if (env) {
1090 env->current_tb = saved_tb;
1091 if (env->interrupt_request && env->current_tb)
1092 cpu_interrupt(env, env->interrupt_request);
1093 }
bellard9fa3e852004-01-04 18:06:42 +00001094 }
1095 tb = tb_next;
1096 }
1097#if !defined(CONFIG_USER_ONLY)
1098 /* if no code remaining, no need to continue to use slow writes */
1099 if (!p->first_tb) {
1100 invalidate_page_bitmap(p);
bellardd720b932004-04-25 17:57:43 +00001101 if (is_cpu_write_access) {
pbrook2e70f6e2008-06-29 01:03:05 +00001102 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
bellardd720b932004-04-25 17:57:43 +00001103 }
1104 }
1105#endif
1106#ifdef TARGET_HAS_PRECISE_SMC
1107 if (current_tb_modified) {
1108 /* we generate a block containing just the instruction
1109 modifying the memory. It will ensure that it cannot modify
1110 itself */
bellardea1c1802004-06-14 18:56:36 +00001111 env->current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001112 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
bellardd720b932004-04-25 17:57:43 +00001113 cpu_resume_from_signal(env, NULL);
bellard9fa3e852004-01-04 18:06:42 +00001114 }
1115#endif
1116}
1117
1118/* len must be <= 8 and start must be a multiple of len */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001119static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
bellard9fa3e852004-01-04 18:06:42 +00001120{
1121 PageDesc *p;
1122 int offset, b;
bellard59817cc2004-02-16 22:01:13 +00001123#if 0
bellarda4193c82004-06-03 14:01:43 +00001124 if (1) {
aliguori93fcfe32009-01-15 22:34:14 +00001125 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1126 cpu_single_env->mem_io_vaddr, len,
1127 cpu_single_env->eip,
1128 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
bellard59817cc2004-02-16 22:01:13 +00001129 }
1130#endif
bellard9fa3e852004-01-04 18:06:42 +00001131 p = page_find(start >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001132 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00001133 return;
1134 if (p->code_bitmap) {
1135 offset = start & ~TARGET_PAGE_MASK;
1136 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1137 if (b & ((1 << len) - 1))
1138 goto do_invalidate;
1139 } else {
1140 do_invalidate:
bellardd720b932004-04-25 17:57:43 +00001141 tb_invalidate_phys_page_range(start, start + len, 1);
bellard9fa3e852004-01-04 18:06:42 +00001142 }
1143}
1144
bellard9fa3e852004-01-04 18:06:42 +00001145#if !defined(CONFIG_SOFTMMU)
Paul Brook41c1b1c2010-03-12 16:54:58 +00001146static void tb_invalidate_phys_page(tb_page_addr_t addr,
bellardd720b932004-04-25 17:57:43 +00001147 unsigned long pc, void *puc)
bellard9fa3e852004-01-04 18:06:42 +00001148{
aliguori6b917542008-11-18 19:46:41 +00001149 TranslationBlock *tb;
bellard9fa3e852004-01-04 18:06:42 +00001150 PageDesc *p;
aliguori6b917542008-11-18 19:46:41 +00001151 int n;
bellardd720b932004-04-25 17:57:43 +00001152#ifdef TARGET_HAS_PRECISE_SMC
aliguori6b917542008-11-18 19:46:41 +00001153 TranslationBlock *current_tb = NULL;
bellardd720b932004-04-25 17:57:43 +00001154 CPUState *env = cpu_single_env;
aliguori6b917542008-11-18 19:46:41 +00001155 int current_tb_modified = 0;
1156 target_ulong current_pc = 0;
1157 target_ulong current_cs_base = 0;
1158 int current_flags = 0;
bellardd720b932004-04-25 17:57:43 +00001159#endif
bellard9fa3e852004-01-04 18:06:42 +00001160
1161 addr &= TARGET_PAGE_MASK;
1162 p = page_find(addr >> TARGET_PAGE_BITS);
ths5fafdf22007-09-16 21:08:06 +00001163 if (!p)
bellardfd6ce8f2003-05-14 19:00:11 +00001164 return;
1165 tb = p->first_tb;
bellardd720b932004-04-25 17:57:43 +00001166#ifdef TARGET_HAS_PRECISE_SMC
1167 if (tb && pc != 0) {
1168 current_tb = tb_find_pc(pc);
1169 }
1170#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001171 while (tb != NULL) {
bellard9fa3e852004-01-04 18:06:42 +00001172 n = (long)tb & 3;
1173 tb = (TranslationBlock *)((long)tb & ~3);
bellardd720b932004-04-25 17:57:43 +00001174#ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb == tb &&
pbrook2e70f6e2008-06-29 01:03:05 +00001176 (current_tb->cflags & CF_COUNT_MASK) != 1) {
bellardd720b932004-04-25 17:57:43 +00001177 /* If we are modifying the current TB, we must stop
1178 its execution. We could be more precise by checking
1179 that the modification is after the current PC, but it
1180 would require a specialized function to partially
1181 restore the CPU state */
ths3b46e622007-09-17 08:09:54 +00001182
bellardd720b932004-04-25 17:57:43 +00001183 current_tb_modified = 1;
Stefan Weil618ba8e2011-04-18 06:39:53 +00001184 cpu_restore_state(current_tb, env, pc);
aliguori6b917542008-11-18 19:46:41 +00001185 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1186 &current_flags);
bellardd720b932004-04-25 17:57:43 +00001187 }
1188#endif /* TARGET_HAS_PRECISE_SMC */
bellard9fa3e852004-01-04 18:06:42 +00001189 tb_phys_invalidate(tb, addr);
1190 tb = tb->page_next[n];
bellardfd6ce8f2003-05-14 19:00:11 +00001191 }
1192 p->first_tb = NULL;
bellardd720b932004-04-25 17:57:43 +00001193#ifdef TARGET_HAS_PRECISE_SMC
1194 if (current_tb_modified) {
1195 /* we generate a block containing just the instruction
1196 modifying the memory. It will ensure that it cannot modify
1197 itself */
bellardea1c1802004-06-14 18:56:36 +00001198 env->current_tb = NULL;
pbrook2e70f6e2008-06-29 01:03:05 +00001199 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
bellardd720b932004-04-25 17:57:43 +00001200 cpu_resume_from_signal(env, puc);
1201 }
1202#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001203}
bellard9fa3e852004-01-04 18:06:42 +00001204#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001205
1206/* add the tb in the target page and protect it if necessary */
ths5fafdf22007-09-16 21:08:06 +00001207static inline void tb_alloc_page(TranslationBlock *tb,
Paul Brook41c1b1c2010-03-12 16:54:58 +00001208 unsigned int n, tb_page_addr_t page_addr)
bellardfd6ce8f2003-05-14 19:00:11 +00001209{
1210 PageDesc *p;
bellard9fa3e852004-01-04 18:06:42 +00001211 TranslationBlock *last_first_tb;
bellardfd6ce8f2003-05-14 19:00:11 +00001212
bellard9fa3e852004-01-04 18:06:42 +00001213 tb->page_addr[n] = page_addr;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001214 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
bellard9fa3e852004-01-04 18:06:42 +00001215 tb->page_next[n] = p->first_tb;
1216 last_first_tb = p->first_tb;
1217 p->first_tb = (TranslationBlock *)((long)tb | n);
1218 invalidate_page_bitmap(p);
1219
bellard107db442004-06-22 18:48:46 +00001220#if defined(TARGET_HAS_SMC) || 1
bellardd720b932004-04-25 17:57:43 +00001221
bellard9fa3e852004-01-04 18:06:42 +00001222#if defined(CONFIG_USER_ONLY)
bellardfd6ce8f2003-05-14 19:00:11 +00001223 if (p->flags & PAGE_WRITE) {
pbrook53a59602006-03-25 19:31:22 +00001224 target_ulong addr;
1225 PageDesc *p2;
bellard9fa3e852004-01-04 18:06:42 +00001226 int prot;
1227
bellardfd6ce8f2003-05-14 19:00:11 +00001228 /* force the host page as non writable (writes will have a
1229 page fault + mprotect overhead) */
pbrook53a59602006-03-25 19:31:22 +00001230 page_addr &= qemu_host_page_mask;
bellardfd6ce8f2003-05-14 19:00:11 +00001231 prot = 0;
pbrook53a59602006-03-25 19:31:22 +00001232 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1233 addr += TARGET_PAGE_SIZE) {
1234
1235 p2 = page_find (addr >> TARGET_PAGE_BITS);
1236 if (!p2)
1237 continue;
1238 prot |= p2->flags;
1239 p2->flags &= ~PAGE_WRITE;
pbrook53a59602006-03-25 19:31:22 +00001240 }
ths5fafdf22007-09-16 21:08:06 +00001241 mprotect(g2h(page_addr), qemu_host_page_size,
bellardfd6ce8f2003-05-14 19:00:11 +00001242 (prot & PAGE_BITS) & ~PAGE_WRITE);
1243#ifdef DEBUG_TB_INVALIDATE
blueswir1ab3d1722007-11-04 07:31:40 +00001244 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
pbrook53a59602006-03-25 19:31:22 +00001245 page_addr);
bellardfd6ce8f2003-05-14 19:00:11 +00001246#endif
bellardfd6ce8f2003-05-14 19:00:11 +00001247 }
bellard9fa3e852004-01-04 18:06:42 +00001248#else
1249 /* if some code is already present, then the pages are already
1250 protected. So we handle the case where only the first TB is
1251 allocated in a physical page */
1252 if (!last_first_tb) {
bellard6a00d602005-11-21 23:25:50 +00001253 tlb_protect_code(page_addr);
bellard9fa3e852004-01-04 18:06:42 +00001254 }
1255#endif
bellardd720b932004-04-25 17:57:43 +00001256
1257#endif /* TARGET_HAS_SMC */
bellardfd6ce8f2003-05-14 19:00:11 +00001258}
1259
bellard9fa3e852004-01-04 18:06:42 +00001260/* add a new TB and link it to the physical page tables. phys_page2 is
1261 (-1) to indicate that only one page contains the TB. */
Paul Brook41c1b1c2010-03-12 16:54:58 +00001262void tb_link_page(TranslationBlock *tb,
1263 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
bellardd4e81642003-05-25 16:46:15 +00001264{
bellard9fa3e852004-01-04 18:06:42 +00001265 unsigned int h;
1266 TranslationBlock **ptb;
1267
pbrookc8a706f2008-06-02 16:16:42 +00001268 /* Grab the mmap lock to stop another thread invalidating this TB
1269 before we are done. */
1270 mmap_lock();
bellard9fa3e852004-01-04 18:06:42 +00001271 /* add in the physical hash table */
1272 h = tb_phys_hash_func(phys_pc);
1273 ptb = &tb_phys_hash[h];
1274 tb->phys_hash_next = *ptb;
1275 *ptb = tb;
bellardfd6ce8f2003-05-14 19:00:11 +00001276
1277 /* add in the page list */
bellard9fa3e852004-01-04 18:06:42 +00001278 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1279 if (phys_page2 != -1)
1280 tb_alloc_page(tb, 1, phys_page2);
1281 else
1282 tb->page_addr[1] = -1;
bellard9fa3e852004-01-04 18:06:42 +00001283
bellardd4e81642003-05-25 16:46:15 +00001284 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1285 tb->jmp_next[0] = NULL;
1286 tb->jmp_next[1] = NULL;
1287
1288 /* init original jump addresses */
1289 if (tb->tb_next_offset[0] != 0xffff)
1290 tb_reset_jump(tb, 0);
1291 if (tb->tb_next_offset[1] != 0xffff)
1292 tb_reset_jump(tb, 1);
bellard8a40a182005-11-20 10:35:40 +00001293
1294#ifdef DEBUG_TB_CHECK
1295 tb_page_check();
1296#endif
pbrookc8a706f2008-06-02 16:16:42 +00001297 mmap_unlock();
bellardfd6ce8f2003-05-14 19:00:11 +00001298}
1299
bellarda513fe12003-05-27 23:29:48 +00001300/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1301 tb[1].tc_ptr. Return NULL if not found */
1302TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1303{
1304 int m_min, m_max, m;
1305 unsigned long v;
1306 TranslationBlock *tb;
1307
1308 if (nb_tbs <= 0)
1309 return NULL;
1310 if (tc_ptr < (unsigned long)code_gen_buffer ||
1311 tc_ptr >= (unsigned long)code_gen_ptr)
1312 return NULL;
1313 /* binary search (cf Knuth) */
1314 m_min = 0;
1315 m_max = nb_tbs - 1;
1316 while (m_min <= m_max) {
1317 m = (m_min + m_max) >> 1;
1318 tb = &tbs[m];
1319 v = (unsigned long)tb->tc_ptr;
1320 if (v == tc_ptr)
1321 return tb;
1322 else if (tc_ptr < v) {
1323 m_max = m - 1;
1324 } else {
1325 m_min = m + 1;
1326 }
ths5fafdf22007-09-16 21:08:06 +00001327 }
bellarda513fe12003-05-27 23:29:48 +00001328 return &tbs[m_max];
1329}
bellard75012672003-06-21 13:11:07 +00001330
bellardea041c02003-06-25 16:16:50 +00001331static void tb_reset_jump_recursive(TranslationBlock *tb);
1332
1333static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1334{
1335 TranslationBlock *tb1, *tb_next, **ptb;
1336 unsigned int n1;
1337
1338 tb1 = tb->jmp_next[n];
1339 if (tb1 != NULL) {
1340 /* find head of list */
1341 for(;;) {
1342 n1 = (long)tb1 & 3;
1343 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1344 if (n1 == 2)
1345 break;
1346 tb1 = tb1->jmp_next[n1];
1347 }
1348 /* we are now sure now that tb jumps to tb1 */
1349 tb_next = tb1;
1350
1351 /* remove tb from the jmp_first list */
1352 ptb = &tb_next->jmp_first;
1353 for(;;) {
1354 tb1 = *ptb;
1355 n1 = (long)tb1 & 3;
1356 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1357 if (n1 == n && tb1 == tb)
1358 break;
1359 ptb = &tb1->jmp_next[n1];
1360 }
1361 *ptb = tb->jmp_next[n];
1362 tb->jmp_next[n] = NULL;
ths3b46e622007-09-17 08:09:54 +00001363
bellardea041c02003-06-25 16:16:50 +00001364 /* suppress the jump to next tb in generated code */
1365 tb_reset_jump(tb, n);
1366
bellard01243112004-01-04 15:48:17 +00001367 /* suppress jumps in the tb on which we could have jumped */
bellardea041c02003-06-25 16:16:50 +00001368 tb_reset_jump_recursive(tb_next);
1369 }
1370}
1371
1372static void tb_reset_jump_recursive(TranslationBlock *tb)
1373{
1374 tb_reset_jump_recursive2(tb, 0);
1375 tb_reset_jump_recursive2(tb, 1);
1376}
1377
bellard1fddef42005-04-17 19:16:13 +00001378#if defined(TARGET_HAS_ICE)
Paul Brook94df27f2010-02-28 23:47:45 +00001379#if defined(CONFIG_USER_ONLY)
1380static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1381{
1382 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1383}
1384#else
bellardd720b932004-04-25 17:57:43 +00001385static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1386{
Anthony Liguoric227f092009-10-01 16:12:16 -05001387 target_phys_addr_t addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00001388 target_ulong pd;
Anthony Liguoric227f092009-10-01 16:12:16 -05001389 ram_addr_t ram_addr;
pbrookc2f07f82006-04-08 17:14:56 +00001390 PhysPageDesc *p;
bellardd720b932004-04-25 17:57:43 +00001391
pbrookc2f07f82006-04-08 17:14:56 +00001392 addr = cpu_get_phys_page_debug(env, pc);
1393 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1394 if (!p) {
1395 pd = IO_MEM_UNASSIGNED;
1396 } else {
1397 pd = p->phys_offset;
1398 }
1399 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
pbrook706cd4b2006-04-08 17:36:21 +00001400 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
bellardd720b932004-04-25 17:57:43 +00001401}
bellardc27004e2005-01-03 23:35:10 +00001402#endif
Paul Brook94df27f2010-02-28 23:47:45 +00001403#endif /* TARGET_HAS_ICE */
bellardd720b932004-04-25 17:57:43 +00001404
Paul Brookc527ee82010-03-01 03:31:14 +00001405#if defined(CONFIG_USER_ONLY)
1406void cpu_watchpoint_remove_all(CPUState *env, int mask)
1407
1408{
1409}
1410
1411int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1412 int flags, CPUWatchpoint **watchpoint)
1413{
1414 return -ENOSYS;
1415}
1416#else
pbrook6658ffb2007-03-16 23:58:11 +00001417/* Add a watchpoint. */
aliguoria1d1bb32008-11-18 20:07:32 +00001418int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1419 int flags, CPUWatchpoint **watchpoint)
pbrook6658ffb2007-03-16 23:58:11 +00001420{
aliguorib4051332008-11-18 20:14:20 +00001421 target_ulong len_mask = ~(len - 1);
aliguoric0ce9982008-11-25 22:13:57 +00001422 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +00001423
aliguorib4051332008-11-18 20:14:20 +00001424 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1425 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1426 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1427 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1428 return -EINVAL;
1429 }
aliguoria1d1bb32008-11-18 20:07:32 +00001430 wp = qemu_malloc(sizeof(*wp));
pbrook6658ffb2007-03-16 23:58:11 +00001431
aliguoria1d1bb32008-11-18 20:07:32 +00001432 wp->vaddr = addr;
aliguorib4051332008-11-18 20:14:20 +00001433 wp->len_mask = len_mask;
aliguoria1d1bb32008-11-18 20:07:32 +00001434 wp->flags = flags;
1435
aliguori2dc9f412008-11-18 20:56:59 +00001436 /* keep all GDB-injected watchpoints in front */
aliguoric0ce9982008-11-25 22:13:57 +00001437 if (flags & BP_GDB)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001438 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
aliguoric0ce9982008-11-25 22:13:57 +00001439 else
Blue Swirl72cf2d42009-09-12 07:36:22 +00001440 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
aliguoria1d1bb32008-11-18 20:07:32 +00001441
pbrook6658ffb2007-03-16 23:58:11 +00001442 tlb_flush_page(env, addr);
aliguoria1d1bb32008-11-18 20:07:32 +00001443
1444 if (watchpoint)
1445 *watchpoint = wp;
1446 return 0;
pbrook6658ffb2007-03-16 23:58:11 +00001447}
1448
aliguoria1d1bb32008-11-18 20:07:32 +00001449/* Remove a specific watchpoint. */
1450int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1451 int flags)
pbrook6658ffb2007-03-16 23:58:11 +00001452{
aliguorib4051332008-11-18 20:14:20 +00001453 target_ulong len_mask = ~(len - 1);
aliguoria1d1bb32008-11-18 20:07:32 +00001454 CPUWatchpoint *wp;
pbrook6658ffb2007-03-16 23:58:11 +00001455
Blue Swirl72cf2d42009-09-12 07:36:22 +00001456 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguorib4051332008-11-18 20:14:20 +00001457 if (addr == wp->vaddr && len_mask == wp->len_mask
aliguori6e140f22008-11-18 20:37:55 +00001458 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
aliguoria1d1bb32008-11-18 20:07:32 +00001459 cpu_watchpoint_remove_by_ref(env, wp);
pbrook6658ffb2007-03-16 23:58:11 +00001460 return 0;
1461 }
1462 }
aliguoria1d1bb32008-11-18 20:07:32 +00001463 return -ENOENT;
pbrook6658ffb2007-03-16 23:58:11 +00001464}
1465
aliguoria1d1bb32008-11-18 20:07:32 +00001466/* Remove a specific watchpoint by reference. */
1467void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1468{
Blue Swirl72cf2d42009-09-12 07:36:22 +00001469 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
edgar_igl7d03f822008-05-17 18:58:29 +00001470
aliguoria1d1bb32008-11-18 20:07:32 +00001471 tlb_flush_page(env, watchpoint->vaddr);
1472
1473 qemu_free(watchpoint);
edgar_igl7d03f822008-05-17 18:58:29 +00001474}
1475
aliguoria1d1bb32008-11-18 20:07:32 +00001476/* Remove all matching watchpoints. */
1477void cpu_watchpoint_remove_all(CPUState *env, int mask)
1478{
aliguoric0ce9982008-11-25 22:13:57 +00001479 CPUWatchpoint *wp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +00001480
Blue Swirl72cf2d42009-09-12 07:36:22 +00001481 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
aliguoria1d1bb32008-11-18 20:07:32 +00001482 if (wp->flags & mask)
1483 cpu_watchpoint_remove_by_ref(env, wp);
aliguoric0ce9982008-11-25 22:13:57 +00001484 }
aliguoria1d1bb32008-11-18 20:07:32 +00001485}
Paul Brookc527ee82010-03-01 03:31:14 +00001486#endif
aliguoria1d1bb32008-11-18 20:07:32 +00001487
1488/* Add a breakpoint. */
1489int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1490 CPUBreakpoint **breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +00001491{
bellard1fddef42005-04-17 19:16:13 +00001492#if defined(TARGET_HAS_ICE)
aliguoric0ce9982008-11-25 22:13:57 +00001493 CPUBreakpoint *bp;
ths3b46e622007-09-17 08:09:54 +00001494
aliguoria1d1bb32008-11-18 20:07:32 +00001495 bp = qemu_malloc(sizeof(*bp));
aliguoria1d1bb32008-11-18 20:07:32 +00001496
1497 bp->pc = pc;
1498 bp->flags = flags;
1499
aliguori2dc9f412008-11-18 20:56:59 +00001500 /* keep all GDB-injected breakpoints in front */
aliguoric0ce9982008-11-25 22:13:57 +00001501 if (flags & BP_GDB)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001502 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
aliguoric0ce9982008-11-25 22:13:57 +00001503 else
Blue Swirl72cf2d42009-09-12 07:36:22 +00001504 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
aliguoria1d1bb32008-11-18 20:07:32 +00001505
1506 breakpoint_invalidate(env, pc);
1507
1508 if (breakpoint)
1509 *breakpoint = bp;
1510 return 0;
1511#else
1512 return -ENOSYS;
1513#endif
1514}
1515
1516/* Remove a specific breakpoint. */
1517int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1518{
1519#if defined(TARGET_HAS_ICE)
1520 CPUBreakpoint *bp;
1521
Blue Swirl72cf2d42009-09-12 07:36:22 +00001522 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +00001523 if (bp->pc == pc && bp->flags == flags) {
1524 cpu_breakpoint_remove_by_ref(env, bp);
bellard4c3a88a2003-07-26 12:06:08 +00001525 return 0;
aliguoria1d1bb32008-11-18 20:07:32 +00001526 }
bellard4c3a88a2003-07-26 12:06:08 +00001527 }
aliguoria1d1bb32008-11-18 20:07:32 +00001528 return -ENOENT;
bellard4c3a88a2003-07-26 12:06:08 +00001529#else
aliguoria1d1bb32008-11-18 20:07:32 +00001530 return -ENOSYS;
bellard4c3a88a2003-07-26 12:06:08 +00001531#endif
1532}
1533
aliguoria1d1bb32008-11-18 20:07:32 +00001534/* Remove a specific breakpoint by reference. */
1535void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
bellard4c3a88a2003-07-26 12:06:08 +00001536{
bellard1fddef42005-04-17 19:16:13 +00001537#if defined(TARGET_HAS_ICE)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001538 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
bellardd720b932004-04-25 17:57:43 +00001539
aliguoria1d1bb32008-11-18 20:07:32 +00001540 breakpoint_invalidate(env, breakpoint->pc);
1541
1542 qemu_free(breakpoint);
1543#endif
1544}
1545
1546/* Remove all matching breakpoints. */
1547void cpu_breakpoint_remove_all(CPUState *env, int mask)
1548{
1549#if defined(TARGET_HAS_ICE)
aliguoric0ce9982008-11-25 22:13:57 +00001550 CPUBreakpoint *bp, *next;
aliguoria1d1bb32008-11-18 20:07:32 +00001551
Blue Swirl72cf2d42009-09-12 07:36:22 +00001552 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
aliguoria1d1bb32008-11-18 20:07:32 +00001553 if (bp->flags & mask)
1554 cpu_breakpoint_remove_by_ref(env, bp);
aliguoric0ce9982008-11-25 22:13:57 +00001555 }
bellard4c3a88a2003-07-26 12:06:08 +00001556#endif
1557}
1558
bellardc33a3462003-07-29 20:50:33 +00001559/* enable or disable single step mode. EXCP_DEBUG is returned by the
1560 CPU loop after each instruction */
1561void cpu_single_step(CPUState *env, int enabled)
1562{
bellard1fddef42005-04-17 19:16:13 +00001563#if defined(TARGET_HAS_ICE)
bellardc33a3462003-07-29 20:50:33 +00001564 if (env->singlestep_enabled != enabled) {
1565 env->singlestep_enabled = enabled;
aliguorie22a25c2009-03-12 20:12:48 +00001566 if (kvm_enabled())
1567 kvm_update_guest_debug(env, 0);
1568 else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +01001569 /* must flush all the translated code to avoid inconsistencies */
aliguorie22a25c2009-03-12 20:12:48 +00001570 /* XXX: only flush what is necessary */
1571 tb_flush(env);
1572 }
bellardc33a3462003-07-29 20:50:33 +00001573 }
1574#endif
1575}
1576
bellard34865132003-10-05 14:28:56 +00001577/* enable or disable low levels log */
1578void cpu_set_log(int log_flags)
1579{
1580 loglevel = log_flags;
1581 if (loglevel && !logfile) {
pbrook11fcfab2007-07-01 18:21:11 +00001582 logfile = fopen(logfilename, log_append ? "a" : "w");
bellard34865132003-10-05 14:28:56 +00001583 if (!logfile) {
1584 perror(logfilename);
1585 _exit(1);
1586 }
bellard9fa3e852004-01-04 18:06:42 +00001587#if !defined(CONFIG_SOFTMMU)
1588 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1589 {
blueswir1b55266b2008-09-20 08:07:15 +00001590 static char logfile_buf[4096];
bellard9fa3e852004-01-04 18:06:42 +00001591 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1592 }
Filip Navarabf65f532009-07-27 10:02:04 -05001593#elif !defined(_WIN32)
1594 /* Win32 doesn't support line-buffering and requires size >= 2 */
bellard34865132003-10-05 14:28:56 +00001595 setvbuf(logfile, NULL, _IOLBF, 0);
bellard9fa3e852004-01-04 18:06:42 +00001596#endif
pbrooke735b912007-06-30 13:53:24 +00001597 log_append = 1;
1598 }
1599 if (!loglevel && logfile) {
1600 fclose(logfile);
1601 logfile = NULL;
bellard34865132003-10-05 14:28:56 +00001602 }
1603}
1604
1605void cpu_set_log_filename(const char *filename)
1606{
1607 logfilename = strdup(filename);
pbrooke735b912007-06-30 13:53:24 +00001608 if (logfile) {
1609 fclose(logfile);
1610 logfile = NULL;
1611 }
1612 cpu_set_log(loglevel);
bellard34865132003-10-05 14:28:56 +00001613}
bellardc33a3462003-07-29 20:50:33 +00001614
aurel323098dba2009-03-07 21:28:24 +00001615static void cpu_unlink_tb(CPUState *env)
bellardea041c02003-06-25 16:16:50 +00001616{
pbrookd5975362008-06-07 20:50:51 +00001617 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1618 problem and hope the cpu will stop of its own accord. For userspace
1619 emulation this often isn't actually as bad as it sounds. Often
1620 signals are used primarily to interrupt blocking syscalls. */
aurel323098dba2009-03-07 21:28:24 +00001621 TranslationBlock *tb;
Anthony Liguoric227f092009-10-01 16:12:16 -05001622 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
aurel323098dba2009-03-07 21:28:24 +00001623
Riku Voipiocab1b4b2010-01-20 12:56:27 +02001624 spin_lock(&interrupt_lock);
aurel323098dba2009-03-07 21:28:24 +00001625 tb = env->current_tb;
1626 /* if the cpu is currently executing code, we must unlink it and
1627 all the potentially executing TB */
Riku Voipiof76cfe52009-12-04 15:16:30 +02001628 if (tb) {
aurel323098dba2009-03-07 21:28:24 +00001629 env->current_tb = NULL;
1630 tb_reset_jump_recursive(tb);
aurel323098dba2009-03-07 21:28:24 +00001631 }
Riku Voipiocab1b4b2010-01-20 12:56:27 +02001632 spin_unlock(&interrupt_lock);
aurel323098dba2009-03-07 21:28:24 +00001633}
1634
Jan Kiszka97ffbd82011-04-13 01:32:56 +02001635#ifndef CONFIG_USER_ONLY
aurel323098dba2009-03-07 21:28:24 +00001636/* mask must never be zero, except for A20 change call */
Jan Kiszkaec6959d2011-04-13 01:32:56 +02001637static void tcg_handle_interrupt(CPUState *env, int mask)
aurel323098dba2009-03-07 21:28:24 +00001638{
1639 int old_mask;
1640
1641 old_mask = env->interrupt_request;
1642 env->interrupt_request |= mask;
1643
aliguori8edac962009-04-24 18:03:45 +00001644 /*
1645 * If called from iothread context, wake the target cpu in
1646 * case its halted.
1647 */
Jan Kiszkab7680cb2011-03-12 17:43:51 +01001648 if (!qemu_cpu_is_self(env)) {
aliguori8edac962009-04-24 18:03:45 +00001649 qemu_cpu_kick(env);
1650 return;
1651 }
aliguori8edac962009-04-24 18:03:45 +00001652
pbrook2e70f6e2008-06-29 01:03:05 +00001653 if (use_icount) {
pbrook266910c2008-07-09 15:31:50 +00001654 env->icount_decr.u16.high = 0xffff;
pbrook2e70f6e2008-06-29 01:03:05 +00001655 if (!can_do_io(env)
aurel32be214e62009-03-06 21:48:00 +00001656 && (mask & ~old_mask) != 0) {
pbrook2e70f6e2008-06-29 01:03:05 +00001657 cpu_abort(env, "Raised interrupt while not in I/O function");
1658 }
pbrook2e70f6e2008-06-29 01:03:05 +00001659 } else {
aurel323098dba2009-03-07 21:28:24 +00001660 cpu_unlink_tb(env);
bellardea041c02003-06-25 16:16:50 +00001661 }
1662}
1663
Jan Kiszkaec6959d2011-04-13 01:32:56 +02001664CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1665
Jan Kiszka97ffbd82011-04-13 01:32:56 +02001666#else /* CONFIG_USER_ONLY */
1667
1668void cpu_interrupt(CPUState *env, int mask)
1669{
1670 env->interrupt_request |= mask;
1671 cpu_unlink_tb(env);
1672}
1673#endif /* CONFIG_USER_ONLY */
1674
bellardb54ad042004-05-20 13:42:52 +00001675void cpu_reset_interrupt(CPUState *env, int mask)
1676{
1677 env->interrupt_request &= ~mask;
1678}
1679
aurel323098dba2009-03-07 21:28:24 +00001680void cpu_exit(CPUState *env)
1681{
1682 env->exit_request = 1;
1683 cpu_unlink_tb(env);
1684}
1685
blueswir1c7cd6a32008-10-02 18:27:46 +00001686const CPULogItem cpu_log_items[] = {
ths5fafdf22007-09-16 21:08:06 +00001687 { CPU_LOG_TB_OUT_ASM, "out_asm",
bellardf193c792004-03-21 17:06:25 +00001688 "show generated host assembly code for each compiled TB" },
1689 { CPU_LOG_TB_IN_ASM, "in_asm",
1690 "show target assembly code for each compiled TB" },
ths5fafdf22007-09-16 21:08:06 +00001691 { CPU_LOG_TB_OP, "op",
bellard57fec1f2008-02-01 10:50:11 +00001692 "show micro ops for each compiled TB" },
bellardf193c792004-03-21 17:06:25 +00001693 { CPU_LOG_TB_OP_OPT, "op_opt",
blueswir1e01a1152008-03-14 17:37:11 +00001694 "show micro ops "
1695#ifdef TARGET_I386
1696 "before eflags optimization and "
bellardf193c792004-03-21 17:06:25 +00001697#endif
blueswir1e01a1152008-03-14 17:37:11 +00001698 "after liveness analysis" },
bellardf193c792004-03-21 17:06:25 +00001699 { CPU_LOG_INT, "int",
1700 "show interrupts/exceptions in short format" },
1701 { CPU_LOG_EXEC, "exec",
1702 "show trace before each executed TB (lots of logs)" },
bellard9fddaa02004-05-21 12:59:32 +00001703 { CPU_LOG_TB_CPU, "cpu",
thse91c8a72007-06-03 13:35:16 +00001704 "show CPU state before block translation" },
bellardf193c792004-03-21 17:06:25 +00001705#ifdef TARGET_I386
1706 { CPU_LOG_PCALL, "pcall",
1707 "show protected mode far calls/returns/exceptions" },
aliguorieca1bdf2009-01-26 19:54:31 +00001708 { CPU_LOG_RESET, "cpu_reset",
1709 "show CPU state before CPU resets" },
bellardf193c792004-03-21 17:06:25 +00001710#endif
bellard8e3a9fd2004-10-09 17:32:58 +00001711#ifdef DEBUG_IOPORT
bellardfd872592004-05-12 19:11:15 +00001712 { CPU_LOG_IOPORT, "ioport",
1713 "show all i/o ports accesses" },
bellard8e3a9fd2004-10-09 17:32:58 +00001714#endif
bellardf193c792004-03-21 17:06:25 +00001715 { 0, NULL, NULL },
1716};
1717
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001718#ifndef CONFIG_USER_ONLY
1719static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1720 = QLIST_HEAD_INITIALIZER(memory_client_list);
1721
1722static void cpu_notify_set_memory(target_phys_addr_t start_addr,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001723 ram_addr_t size,
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03001724 ram_addr_t phys_offset,
1725 bool log_dirty)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001726{
1727 CPUPhysMemoryClient *client;
1728 QLIST_FOREACH(client, &memory_client_list, list) {
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03001729 client->set_memory(client, start_addr, size, phys_offset, log_dirty);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001730 }
1731}
1732
1733static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001734 target_phys_addr_t end)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001735{
1736 CPUPhysMemoryClient *client;
1737 QLIST_FOREACH(client, &memory_client_list, list) {
1738 int r = client->sync_dirty_bitmap(client, start, end);
1739 if (r < 0)
1740 return r;
1741 }
1742 return 0;
1743}
1744
1745static int cpu_notify_migration_log(int enable)
1746{
1747 CPUPhysMemoryClient *client;
1748 QLIST_FOREACH(client, &memory_client_list, list) {
1749 int r = client->migration_log(client, enable);
1750 if (r < 0)
1751 return r;
1752 }
1753 return 0;
1754}
1755
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001756/* The l1_phys_map provides the upper P_L1_BITs of the guest physical
1757 * address. Each intermediate table provides the next L2_BITs of guest
1758 * physical address space. The number of levels vary based on host and
1759 * guest configuration, making it efficient to build the final guest
1760 * physical address by seeding the L1 offset and shifting and adding in
1761 * each L2 offset as we recurse through them. */
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001762static void phys_page_for_each_1(CPUPhysMemoryClient *client,
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001763 int level, void **lp, target_phys_addr_t addr)
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001764{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001765 int i;
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001766
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001767 if (*lp == NULL) {
1768 return;
1769 }
1770 if (level == 0) {
1771 PhysPageDesc *pd = *lp;
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001772 addr <<= L2_BITS + TARGET_PAGE_BITS;
Paul Brook7296aba2010-03-14 14:58:46 +00001773 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001774 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001775 client->set_memory(client, addr | i << TARGET_PAGE_BITS,
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03001776 TARGET_PAGE_SIZE, pd[i].phys_offset, false);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001777 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001778 }
1779 } else {
1780 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00001781 for (i = 0; i < L2_SIZE; ++i) {
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001782 phys_page_for_each_1(client, level - 1, pp + i,
1783 (addr << L2_BITS) | i);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001784 }
1785 }
1786}
1787
1788static void phys_page_for_each(CPUPhysMemoryClient *client)
1789{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08001790 int i;
1791 for (i = 0; i < P_L1_SIZE; ++i) {
1792 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
Alex Williamson8d4c78e2011-05-03 12:36:46 -06001793 l1_phys_map + i, i);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001794 }
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02001795}
1796
1797void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1798{
1799 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1800 phys_page_for_each(client);
1801}
1802
1803void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1804{
1805 QLIST_REMOVE(client, list);
1806}
1807#endif
1808
bellardf193c792004-03-21 17:06:25 +00001809static int cmp1(const char *s1, int n, const char *s2)
1810{
1811 if (strlen(s2) != n)
1812 return 0;
1813 return memcmp(s1, s2, n) == 0;
1814}
ths3b46e622007-09-17 08:09:54 +00001815
bellardf193c792004-03-21 17:06:25 +00001816/* takes a comma separated list of log masks. Return 0 if error. */
1817int cpu_str_to_log_mask(const char *str)
1818{
blueswir1c7cd6a32008-10-02 18:27:46 +00001819 const CPULogItem *item;
bellardf193c792004-03-21 17:06:25 +00001820 int mask;
1821 const char *p, *p1;
1822
1823 p = str;
1824 mask = 0;
1825 for(;;) {
1826 p1 = strchr(p, ',');
1827 if (!p1)
1828 p1 = p + strlen(p);
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001829 if(cmp1(p,p1-p,"all")) {
1830 for(item = cpu_log_items; item->mask != 0; item++) {
1831 mask |= item->mask;
1832 }
1833 } else {
1834 for(item = cpu_log_items; item->mask != 0; item++) {
1835 if (cmp1(p, p1 - p, item->name))
1836 goto found;
1837 }
1838 return 0;
bellardf193c792004-03-21 17:06:25 +00001839 }
bellardf193c792004-03-21 17:06:25 +00001840 found:
1841 mask |= item->mask;
1842 if (*p1 != ',')
1843 break;
1844 p = p1 + 1;
1845 }
1846 return mask;
1847}
bellardea041c02003-06-25 16:16:50 +00001848
bellard75012672003-06-21 13:11:07 +00001849void cpu_abort(CPUState *env, const char *fmt, ...)
1850{
1851 va_list ap;
pbrook493ae1f2007-11-23 16:53:59 +00001852 va_list ap2;
bellard75012672003-06-21 13:11:07 +00001853
1854 va_start(ap, fmt);
pbrook493ae1f2007-11-23 16:53:59 +00001855 va_copy(ap2, ap);
bellard75012672003-06-21 13:11:07 +00001856 fprintf(stderr, "qemu: fatal: ");
1857 vfprintf(stderr, fmt, ap);
1858 fprintf(stderr, "\n");
1859#ifdef TARGET_I386
bellard7fe48482004-10-09 18:08:01 +00001860 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1861#else
1862 cpu_dump_state(env, stderr, fprintf, 0);
bellard75012672003-06-21 13:11:07 +00001863#endif
aliguori93fcfe32009-01-15 22:34:14 +00001864 if (qemu_log_enabled()) {
1865 qemu_log("qemu: fatal: ");
1866 qemu_log_vprintf(fmt, ap2);
1867 qemu_log("\n");
j_mayerf9373292007-09-29 12:18:20 +00001868#ifdef TARGET_I386
aliguori93fcfe32009-01-15 22:34:14 +00001869 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
j_mayerf9373292007-09-29 12:18:20 +00001870#else
aliguori93fcfe32009-01-15 22:34:14 +00001871 log_cpu_state(env, 0);
j_mayerf9373292007-09-29 12:18:20 +00001872#endif
aliguori31b1a7b2009-01-15 22:35:09 +00001873 qemu_log_flush();
aliguori93fcfe32009-01-15 22:34:14 +00001874 qemu_log_close();
balrog924edca2007-06-10 14:07:13 +00001875 }
pbrook493ae1f2007-11-23 16:53:59 +00001876 va_end(ap2);
j_mayerf9373292007-09-29 12:18:20 +00001877 va_end(ap);
Riku Voipiofd052bf2010-01-25 14:30:49 +02001878#if defined(CONFIG_USER_ONLY)
1879 {
1880 struct sigaction act;
1881 sigfillset(&act.sa_mask);
1882 act.sa_handler = SIG_DFL;
1883 sigaction(SIGABRT, &act, NULL);
1884 }
1885#endif
bellard75012672003-06-21 13:11:07 +00001886 abort();
1887}
1888
thsc5be9f02007-02-28 20:20:53 +00001889CPUState *cpu_copy(CPUState *env)
1890{
ths01ba9812007-12-09 02:22:57 +00001891 CPUState *new_env = cpu_init(env->cpu_model_str);
thsc5be9f02007-02-28 20:20:53 +00001892 CPUState *next_cpu = new_env->next_cpu;
1893 int cpu_index = new_env->cpu_index;
aliguori5a38f082009-01-15 20:16:51 +00001894#if defined(TARGET_HAS_ICE)
1895 CPUBreakpoint *bp;
1896 CPUWatchpoint *wp;
1897#endif
1898
thsc5be9f02007-02-28 20:20:53 +00001899 memcpy(new_env, env, sizeof(CPUState));
aliguori5a38f082009-01-15 20:16:51 +00001900
1901 /* Preserve chaining and index. */
thsc5be9f02007-02-28 20:20:53 +00001902 new_env->next_cpu = next_cpu;
1903 new_env->cpu_index = cpu_index;
aliguori5a38f082009-01-15 20:16:51 +00001904
1905 /* Clone all break/watchpoints.
1906 Note: Once we support ptrace with hw-debug register access, make sure
1907 BP_CPU break/watchpoints are handled correctly on clone. */
Blue Swirl72cf2d42009-09-12 07:36:22 +00001908 QTAILQ_INIT(&env->breakpoints);
1909 QTAILQ_INIT(&env->watchpoints);
aliguori5a38f082009-01-15 20:16:51 +00001910#if defined(TARGET_HAS_ICE)
Blue Swirl72cf2d42009-09-12 07:36:22 +00001911 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
aliguori5a38f082009-01-15 20:16:51 +00001912 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1913 }
Blue Swirl72cf2d42009-09-12 07:36:22 +00001914 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguori5a38f082009-01-15 20:16:51 +00001915 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1916 wp->flags, NULL);
1917 }
1918#endif
1919
thsc5be9f02007-02-28 20:20:53 +00001920 return new_env;
1921}
1922
bellard01243112004-01-04 15:48:17 +00001923#if !defined(CONFIG_USER_ONLY)
1924
edgar_igl5c751e92008-05-06 08:44:21 +00001925static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1926{
1927 unsigned int i;
1928
1929 /* Discard jump cache entries for any tb which might potentially
1930 overlap the flushed page. */
1931 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1932 memset (&env->tb_jmp_cache[i], 0,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001933 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
edgar_igl5c751e92008-05-06 08:44:21 +00001934
1935 i = tb_jmp_cache_hash_page(addr);
1936 memset (&env->tb_jmp_cache[i], 0,
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09001937 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
edgar_igl5c751e92008-05-06 08:44:21 +00001938}
1939
Igor Kovalenko08738982009-07-12 02:15:40 +04001940static CPUTLBEntry s_cputlb_empty_entry = {
1941 .addr_read = -1,
1942 .addr_write = -1,
1943 .addr_code = -1,
1944 .addend = -1,
1945};
1946
bellardee8b7022004-02-03 23:35:10 +00001947/* NOTE: if flush_global is true, also flush global entries (not
1948 implemented yet) */
1949void tlb_flush(CPUState *env, int flush_global)
bellard33417e72003-08-10 21:47:01 +00001950{
bellard33417e72003-08-10 21:47:01 +00001951 int i;
bellard01243112004-01-04 15:48:17 +00001952
bellard9fa3e852004-01-04 18:06:42 +00001953#if defined(DEBUG_TLB)
1954 printf("tlb_flush:\n");
1955#endif
bellard01243112004-01-04 15:48:17 +00001956 /* must reset current TB so that interrupts cannot modify the
1957 links while we are modifying them */
1958 env->current_tb = NULL;
1959
bellard33417e72003-08-10 21:47:01 +00001960 for(i = 0; i < CPU_TLB_SIZE; i++) {
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09001961 int mmu_idx;
1962 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
Igor Kovalenko08738982009-07-12 02:15:40 +04001963 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09001964 }
bellard33417e72003-08-10 21:47:01 +00001965 }
bellard9fa3e852004-01-04 18:06:42 +00001966
bellard8a40a182005-11-20 10:35:40 +00001967 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
bellard9fa3e852004-01-04 18:06:42 +00001968
Paul Brookd4c430a2010-03-17 02:14:28 +00001969 env->tlb_flush_addr = -1;
1970 env->tlb_flush_mask = 0;
bellarde3db7222005-01-26 22:00:47 +00001971 tlb_flush_count++;
bellard33417e72003-08-10 21:47:01 +00001972}
1973
bellard274da6b2004-05-20 21:56:27 +00001974static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
bellard61382a52003-10-27 21:22:23 +00001975{
ths5fafdf22007-09-16 21:08:06 +00001976 if (addr == (tlb_entry->addr_read &
bellard84b7b8e2005-11-28 21:19:04 +00001977 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
ths5fafdf22007-09-16 21:08:06 +00001978 addr == (tlb_entry->addr_write &
bellard84b7b8e2005-11-28 21:19:04 +00001979 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
ths5fafdf22007-09-16 21:08:06 +00001980 addr == (tlb_entry->addr_code &
bellard84b7b8e2005-11-28 21:19:04 +00001981 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
Igor Kovalenko08738982009-07-12 02:15:40 +04001982 *tlb_entry = s_cputlb_empty_entry;
bellard84b7b8e2005-11-28 21:19:04 +00001983 }
bellard61382a52003-10-27 21:22:23 +00001984}
1985
bellard2e126692004-04-25 21:28:44 +00001986void tlb_flush_page(CPUState *env, target_ulong addr)
bellard33417e72003-08-10 21:47:01 +00001987{
bellard8a40a182005-11-20 10:35:40 +00001988 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09001989 int mmu_idx;
bellard01243112004-01-04 15:48:17 +00001990
bellard9fa3e852004-01-04 18:06:42 +00001991#if defined(DEBUG_TLB)
bellard108c49b2005-07-24 12:55:09 +00001992 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
bellard9fa3e852004-01-04 18:06:42 +00001993#endif
Paul Brookd4c430a2010-03-17 02:14:28 +00001994 /* Check if we need to flush due to large pages. */
1995 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1996#if defined(DEBUG_TLB)
1997 printf("tlb_flush_page: forced full flush ("
1998 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1999 env->tlb_flush_addr, env->tlb_flush_mask);
2000#endif
2001 tlb_flush(env, 1);
2002 return;
2003 }
bellard01243112004-01-04 15:48:17 +00002004 /* must reset current TB so that interrupts cannot modify the
2005 links while we are modifying them */
2006 env->current_tb = NULL;
bellard33417e72003-08-10 21:47:01 +00002007
bellard61382a52003-10-27 21:22:23 +00002008 addr &= TARGET_PAGE_MASK;
bellard33417e72003-08-10 21:47:01 +00002009 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002010 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2011 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
bellard01243112004-01-04 15:48:17 +00002012
edgar_igl5c751e92008-05-06 08:44:21 +00002013 tlb_flush_jmp_cache(env, addr);
bellard9fa3e852004-01-04 18:06:42 +00002014}
2015
bellard9fa3e852004-01-04 18:06:42 +00002016/* update the TLBs so that writes to code in the virtual page 'addr'
2017 can be detected */
Anthony Liguoric227f092009-10-01 16:12:16 -05002018static void tlb_protect_code(ram_addr_t ram_addr)
bellard61382a52003-10-27 21:22:23 +00002019{
ths5fafdf22007-09-16 21:08:06 +00002020 cpu_physical_memory_reset_dirty(ram_addr,
bellard6a00d602005-11-21 23:25:50 +00002021 ram_addr + TARGET_PAGE_SIZE,
2022 CODE_DIRTY_FLAG);
bellard9fa3e852004-01-04 18:06:42 +00002023}
2024
bellard9fa3e852004-01-04 18:06:42 +00002025/* update the TLB so that writes in physical page 'phys_addr' are no longer
bellard3a7d9292005-08-21 09:26:42 +00002026 tested for self modifying code */
Anthony Liguoric227f092009-10-01 16:12:16 -05002027static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
bellard3a7d9292005-08-21 09:26:42 +00002028 target_ulong vaddr)
bellard9fa3e852004-01-04 18:06:42 +00002029{
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002030 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
bellard1ccde1c2004-02-06 19:46:14 +00002031}
2032
ths5fafdf22007-09-16 21:08:06 +00002033static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
bellard1ccde1c2004-02-06 19:46:14 +00002034 unsigned long start, unsigned long length)
2035{
2036 unsigned long addr;
bellard84b7b8e2005-11-28 21:19:04 +00002037 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2038 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
bellard1ccde1c2004-02-06 19:46:14 +00002039 if ((addr - start) < length) {
pbrook0f459d12008-06-09 00:20:13 +00002040 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
bellard1ccde1c2004-02-06 19:46:14 +00002041 }
2042 }
2043}
2044
pbrook5579c7f2009-04-11 14:47:08 +00002045/* Note: start and end must be within the same ram block. */
Anthony Liguoric227f092009-10-01 16:12:16 -05002046void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
bellard0a962c02005-02-10 22:00:27 +00002047 int dirty_flags)
bellard1ccde1c2004-02-06 19:46:14 +00002048{
2049 CPUState *env;
bellard4f2ac232004-04-26 19:44:02 +00002050 unsigned long length, start1;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002051 int i;
bellard1ccde1c2004-02-06 19:46:14 +00002052
2053 start &= TARGET_PAGE_MASK;
2054 end = TARGET_PAGE_ALIGN(end);
2055
2056 length = end - start;
2057 if (length == 0)
2058 return;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09002059 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00002060
bellard1ccde1c2004-02-06 19:46:14 +00002061 /* we modify the TLB cache so that the dirty bit will be set again
2062 when accessing the range */
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02002063 start1 = (unsigned long)qemu_safe_ram_ptr(start);
pbrook5579c7f2009-04-11 14:47:08 +00002064 /* Chek that we don't span multiple blocks - this breaks the
2065 address comparisons below. */
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02002066 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
pbrook5579c7f2009-04-11 14:47:08 +00002067 != (end - 1) - start) {
2068 abort();
2069 }
2070
bellard6a00d602005-11-21 23:25:50 +00002071 for(env = first_cpu; env != NULL; env = env->next_cpu) {
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002072 int mmu_idx;
2073 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2074 for(i = 0; i < CPU_TLB_SIZE; i++)
2075 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2076 start1, length);
2077 }
bellard6a00d602005-11-21 23:25:50 +00002078 }
bellard1ccde1c2004-02-06 19:46:14 +00002079}
2080
aliguori74576192008-10-06 14:02:03 +00002081int cpu_physical_memory_set_dirty_tracking(int enable)
2082{
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002083 int ret = 0;
aliguori74576192008-10-06 14:02:03 +00002084 in_migration = enable;
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002085 ret = cpu_notify_migration_log(!!enable);
2086 return ret;
aliguori74576192008-10-06 14:02:03 +00002087}
2088
2089int cpu_physical_memory_get_dirty_tracking(void)
2090{
2091 return in_migration;
2092}
2093
Anthony Liguoric227f092009-10-01 16:12:16 -05002094int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2095 target_phys_addr_t end_addr)
aliguori2bec46d2008-11-24 20:21:41 +00002096{
Michael S. Tsirkin7b8f3b72010-01-27 22:07:21 +02002097 int ret;
Jan Kiszka151f7742009-05-01 20:52:47 +02002098
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002099 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
Jan Kiszka151f7742009-05-01 20:52:47 +02002100 return ret;
aliguori2bec46d2008-11-24 20:21:41 +00002101}
2102
Anthony PERARDe5896b12011-02-07 12:19:23 +01002103int cpu_physical_log_start(target_phys_addr_t start_addr,
2104 ram_addr_t size)
2105{
2106 CPUPhysMemoryClient *client;
2107 QLIST_FOREACH(client, &memory_client_list, list) {
2108 if (client->log_start) {
2109 int r = client->log_start(client, start_addr, size);
2110 if (r < 0) {
2111 return r;
2112 }
2113 }
2114 }
2115 return 0;
2116}
2117
2118int cpu_physical_log_stop(target_phys_addr_t start_addr,
2119 ram_addr_t size)
2120{
2121 CPUPhysMemoryClient *client;
2122 QLIST_FOREACH(client, &memory_client_list, list) {
2123 if (client->log_stop) {
2124 int r = client->log_stop(client, start_addr, size);
2125 if (r < 0) {
2126 return r;
2127 }
2128 }
2129 }
2130 return 0;
2131}
2132
bellard3a7d9292005-08-21 09:26:42 +00002133static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2134{
Anthony Liguoric227f092009-10-01 16:12:16 -05002135 ram_addr_t ram_addr;
pbrook5579c7f2009-04-11 14:47:08 +00002136 void *p;
bellard3a7d9292005-08-21 09:26:42 +00002137
bellard84b7b8e2005-11-28 21:19:04 +00002138 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
pbrook5579c7f2009-04-11 14:47:08 +00002139 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2140 + tlb_entry->addend);
Marcelo Tosattie8902612010-10-11 15:31:19 -03002141 ram_addr = qemu_ram_addr_from_host_nofail(p);
bellard3a7d9292005-08-21 09:26:42 +00002142 if (!cpu_physical_memory_is_dirty(ram_addr)) {
pbrook0f459d12008-06-09 00:20:13 +00002143 tlb_entry->addr_write |= TLB_NOTDIRTY;
bellard3a7d9292005-08-21 09:26:42 +00002144 }
2145 }
2146}
2147
2148/* update the TLB according to the current state of the dirty bits */
2149void cpu_tlb_update_dirty(CPUState *env)
2150{
2151 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002152 int mmu_idx;
2153 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2154 for(i = 0; i < CPU_TLB_SIZE; i++)
2155 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2156 }
bellard3a7d9292005-08-21 09:26:42 +00002157}
2158
pbrook0f459d12008-06-09 00:20:13 +00002159static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002160{
pbrook0f459d12008-06-09 00:20:13 +00002161 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2162 tlb_entry->addr_write = vaddr;
bellard1ccde1c2004-02-06 19:46:14 +00002163}
2164
pbrook0f459d12008-06-09 00:20:13 +00002165/* update the TLB corresponding to virtual page vaddr
2166 so that it is no longer dirty */
2167static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002168{
bellard1ccde1c2004-02-06 19:46:14 +00002169 int i;
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002170 int mmu_idx;
bellard1ccde1c2004-02-06 19:46:14 +00002171
pbrook0f459d12008-06-09 00:20:13 +00002172 vaddr &= TARGET_PAGE_MASK;
bellard1ccde1c2004-02-06 19:46:14 +00002173 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
Isaku Yamahatacfde4bd2009-05-20 11:31:43 +09002174 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2175 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
bellard9fa3e852004-01-04 18:06:42 +00002176}
2177
Paul Brookd4c430a2010-03-17 02:14:28 +00002178/* Our TLB does not support large pages, so remember the area covered by
2179 large pages and trigger a full TLB flush if these are invalidated. */
2180static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2181 target_ulong size)
2182{
2183 target_ulong mask = ~(size - 1);
2184
2185 if (env->tlb_flush_addr == (target_ulong)-1) {
2186 env->tlb_flush_addr = vaddr & mask;
2187 env->tlb_flush_mask = mask;
2188 return;
2189 }
2190 /* Extend the existing region to include the new page.
2191 This is a compromise between unnecessary flushes and the cost
2192 of maintaining a full variable size TLB. */
2193 mask &= env->tlb_flush_mask;
2194 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2195 mask <<= 1;
2196 }
2197 env->tlb_flush_addr &= mask;
2198 env->tlb_flush_mask = mask;
2199}
2200
2201/* Add a new TLB entry. At most one entry for a given virtual address
2202 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2203 supplied size is only used by tlb_flush_page. */
2204void tlb_set_page(CPUState *env, target_ulong vaddr,
2205 target_phys_addr_t paddr, int prot,
2206 int mmu_idx, target_ulong size)
bellard9fa3e852004-01-04 18:06:42 +00002207{
bellard92e873b2004-05-21 14:52:29 +00002208 PhysPageDesc *p;
bellard4f2ac232004-04-26 19:44:02 +00002209 unsigned long pd;
bellard9fa3e852004-01-04 18:06:42 +00002210 unsigned int index;
bellard4f2ac232004-04-26 19:44:02 +00002211 target_ulong address;
pbrook0f459d12008-06-09 00:20:13 +00002212 target_ulong code_address;
Paul Brook355b1942010-04-05 00:28:53 +01002213 unsigned long addend;
bellard84b7b8e2005-11-28 21:19:04 +00002214 CPUTLBEntry *te;
aliguoria1d1bb32008-11-18 20:07:32 +00002215 CPUWatchpoint *wp;
Anthony Liguoric227f092009-10-01 16:12:16 -05002216 target_phys_addr_t iotlb;
bellard9fa3e852004-01-04 18:06:42 +00002217
Paul Brookd4c430a2010-03-17 02:14:28 +00002218 assert(size >= TARGET_PAGE_SIZE);
2219 if (size != TARGET_PAGE_SIZE) {
2220 tlb_add_large_page(env, vaddr, size);
2221 }
bellard92e873b2004-05-21 14:52:29 +00002222 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
bellard9fa3e852004-01-04 18:06:42 +00002223 if (!p) {
2224 pd = IO_MEM_UNASSIGNED;
bellard9fa3e852004-01-04 18:06:42 +00002225 } else {
2226 pd = p->phys_offset;
bellard9fa3e852004-01-04 18:06:42 +00002227 }
2228#if defined(DEBUG_TLB)
Stefan Weil7fd3f492010-09-30 22:39:51 +02002229 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2230 " prot=%x idx=%d pd=0x%08lx\n",
2231 vaddr, paddr, prot, mmu_idx, pd);
bellard9fa3e852004-01-04 18:06:42 +00002232#endif
2233
pbrook0f459d12008-06-09 00:20:13 +00002234 address = vaddr;
2235 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2236 /* IO memory case (romd handled later) */
2237 address |= TLB_MMIO;
2238 }
pbrook5579c7f2009-04-11 14:47:08 +00002239 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
pbrook0f459d12008-06-09 00:20:13 +00002240 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2241 /* Normal RAM. */
2242 iotlb = pd & TARGET_PAGE_MASK;
2243 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2244 iotlb |= IO_MEM_NOTDIRTY;
2245 else
2246 iotlb |= IO_MEM_ROM;
2247 } else {
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002248 /* IO handlers are currently passed a physical address.
pbrook0f459d12008-06-09 00:20:13 +00002249 It would be nice to pass an offset from the base address
2250 of that region. This would avoid having to special case RAM,
2251 and avoid full address decoding in every device.
2252 We can't use the high bits of pd for this because
2253 IO_MEM_ROMD uses these as a ram address. */
pbrook8da3ff12008-12-01 18:59:50 +00002254 iotlb = (pd & ~TARGET_PAGE_MASK);
2255 if (p) {
pbrook8da3ff12008-12-01 18:59:50 +00002256 iotlb += p->region_offset;
2257 } else {
2258 iotlb += paddr;
2259 }
pbrook0f459d12008-06-09 00:20:13 +00002260 }
pbrook6658ffb2007-03-16 23:58:11 +00002261
pbrook0f459d12008-06-09 00:20:13 +00002262 code_address = address;
2263 /* Make accesses to pages with watchpoints go via the
2264 watchpoint trap routines. */
Blue Swirl72cf2d42009-09-12 07:36:22 +00002265 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguoria1d1bb32008-11-18 20:07:32 +00002266 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
Jun Koibf298f82010-05-06 14:36:59 +09002267 /* Avoid trapping reads of pages with a write breakpoint. */
2268 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2269 iotlb = io_mem_watch + paddr;
2270 address |= TLB_MMIO;
2271 break;
2272 }
pbrook6658ffb2007-03-16 23:58:11 +00002273 }
pbrook0f459d12008-06-09 00:20:13 +00002274 }
balrogd79acba2007-06-26 20:01:13 +00002275
pbrook0f459d12008-06-09 00:20:13 +00002276 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2277 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2278 te = &env->tlb_table[mmu_idx][index];
2279 te->addend = addend - vaddr;
2280 if (prot & PAGE_READ) {
2281 te->addr_read = address;
2282 } else {
2283 te->addr_read = -1;
2284 }
edgar_igl5c751e92008-05-06 08:44:21 +00002285
pbrook0f459d12008-06-09 00:20:13 +00002286 if (prot & PAGE_EXEC) {
2287 te->addr_code = code_address;
2288 } else {
2289 te->addr_code = -1;
2290 }
2291 if (prot & PAGE_WRITE) {
2292 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2293 (pd & IO_MEM_ROMD)) {
2294 /* Write access calls the I/O callback. */
2295 te->addr_write = address | TLB_MMIO;
2296 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2297 !cpu_physical_memory_is_dirty(pd)) {
2298 te->addr_write = address | TLB_NOTDIRTY;
bellard84b7b8e2005-11-28 21:19:04 +00002299 } else {
pbrook0f459d12008-06-09 00:20:13 +00002300 te->addr_write = address;
bellard9fa3e852004-01-04 18:06:42 +00002301 }
pbrook0f459d12008-06-09 00:20:13 +00002302 } else {
2303 te->addr_write = -1;
bellard9fa3e852004-01-04 18:06:42 +00002304 }
bellard9fa3e852004-01-04 18:06:42 +00002305}
2306
bellard01243112004-01-04 15:48:17 +00002307#else
2308
bellardee8b7022004-02-03 23:35:10 +00002309void tlb_flush(CPUState *env, int flush_global)
bellard01243112004-01-04 15:48:17 +00002310{
2311}
2312
bellard2e126692004-04-25 21:28:44 +00002313void tlb_flush_page(CPUState *env, target_ulong addr)
bellard01243112004-01-04 15:48:17 +00002314{
2315}
2316
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002317/*
2318 * Walks guest process memory "regions" one by one
2319 * and calls callback function 'fn' for each region.
2320 */
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002321
2322struct walk_memory_regions_data
bellard9fa3e852004-01-04 18:06:42 +00002323{
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002324 walk_memory_regions_fn fn;
2325 void *priv;
2326 unsigned long start;
2327 int prot;
2328};
bellard9fa3e852004-01-04 18:06:42 +00002329
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002330static int walk_memory_regions_end(struct walk_memory_regions_data *data,
Paul Brookb480d9b2010-03-12 23:23:29 +00002331 abi_ulong end, int new_prot)
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002332{
2333 if (data->start != -1ul) {
2334 int rc = data->fn(data->priv, data->start, end, data->prot);
2335 if (rc != 0) {
2336 return rc;
bellard9fa3e852004-01-04 18:06:42 +00002337 }
bellard33417e72003-08-10 21:47:01 +00002338 }
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002339
2340 data->start = (new_prot ? end : -1ul);
2341 data->prot = new_prot;
2342
2343 return 0;
2344}
2345
2346static int walk_memory_regions_1(struct walk_memory_regions_data *data,
Paul Brookb480d9b2010-03-12 23:23:29 +00002347 abi_ulong base, int level, void **lp)
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002348{
Paul Brookb480d9b2010-03-12 23:23:29 +00002349 abi_ulong pa;
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002350 int i, rc;
2351
2352 if (*lp == NULL) {
2353 return walk_memory_regions_end(data, base, 0);
2354 }
2355
2356 if (level == 0) {
2357 PageDesc *pd = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00002358 for (i = 0; i < L2_SIZE; ++i) {
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002359 int prot = pd[i].flags;
2360
2361 pa = base | (i << TARGET_PAGE_BITS);
2362 if (prot != data->prot) {
2363 rc = walk_memory_regions_end(data, pa, prot);
2364 if (rc != 0) {
2365 return rc;
2366 }
2367 }
2368 }
2369 } else {
2370 void **pp = *lp;
Paul Brook7296aba2010-03-14 14:58:46 +00002371 for (i = 0; i < L2_SIZE; ++i) {
Paul Brookb480d9b2010-03-12 23:23:29 +00002372 pa = base | ((abi_ulong)i <<
2373 (TARGET_PAGE_BITS + L2_BITS * level));
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002374 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2375 if (rc != 0) {
2376 return rc;
2377 }
2378 }
2379 }
2380
2381 return 0;
2382}
2383
2384int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2385{
2386 struct walk_memory_regions_data data;
2387 unsigned long i;
2388
2389 data.fn = fn;
2390 data.priv = priv;
2391 data.start = -1ul;
2392 data.prot = 0;
2393
2394 for (i = 0; i < V_L1_SIZE; i++) {
Paul Brookb480d9b2010-03-12 23:23:29 +00002395 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
Richard Henderson5cd2c5b2010-03-10 15:53:37 -08002396 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2397 if (rc != 0) {
2398 return rc;
2399 }
2400 }
2401
2402 return walk_memory_regions_end(&data, 0, 0);
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002403}
2404
Paul Brookb480d9b2010-03-12 23:23:29 +00002405static int dump_region(void *priv, abi_ulong start,
2406 abi_ulong end, unsigned long prot)
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002407{
2408 FILE *f = (FILE *)priv;
2409
Paul Brookb480d9b2010-03-12 23:23:29 +00002410 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2411 " "TARGET_ABI_FMT_lx" %c%c%c\n",
Mika Westerbergedf8e2a2009-04-07 09:57:11 +03002412 start, end, end - start,
2413 ((prot & PAGE_READ) ? 'r' : '-'),
2414 ((prot & PAGE_WRITE) ? 'w' : '-'),
2415 ((prot & PAGE_EXEC) ? 'x' : '-'));
2416
2417 return (0);
2418}
2419
2420/* dump memory mappings */
2421void page_dump(FILE *f)
2422{
2423 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2424 "start", "end", "size", "prot");
2425 walk_memory_regions(f, dump_region);
bellard33417e72003-08-10 21:47:01 +00002426}
2427
pbrook53a59602006-03-25 19:31:22 +00002428int page_get_flags(target_ulong address)
bellard33417e72003-08-10 21:47:01 +00002429{
bellard9fa3e852004-01-04 18:06:42 +00002430 PageDesc *p;
2431
2432 p = page_find(address >> TARGET_PAGE_BITS);
bellard33417e72003-08-10 21:47:01 +00002433 if (!p)
bellard9fa3e852004-01-04 18:06:42 +00002434 return 0;
2435 return p->flags;
bellard33417e72003-08-10 21:47:01 +00002436}
2437
Richard Henderson376a7902010-03-10 15:57:04 -08002438/* Modify the flags of a page and invalidate the code if necessary.
2439 The flag PAGE_WRITE_ORG is positioned automatically depending
2440 on PAGE_WRITE. The mmap_lock should already be held. */
pbrook53a59602006-03-25 19:31:22 +00002441void page_set_flags(target_ulong start, target_ulong end, int flags)
bellard9fa3e852004-01-04 18:06:42 +00002442{
Richard Henderson376a7902010-03-10 15:57:04 -08002443 target_ulong addr, len;
bellard9fa3e852004-01-04 18:06:42 +00002444
Richard Henderson376a7902010-03-10 15:57:04 -08002445 /* This function should never be called with addresses outside the
2446 guest address space. If this assert fires, it probably indicates
2447 a missing call to h2g_valid. */
Paul Brookb480d9b2010-03-12 23:23:29 +00002448#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2449 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Richard Henderson376a7902010-03-10 15:57:04 -08002450#endif
2451 assert(start < end);
2452
bellard9fa3e852004-01-04 18:06:42 +00002453 start = start & TARGET_PAGE_MASK;
2454 end = TARGET_PAGE_ALIGN(end);
Richard Henderson376a7902010-03-10 15:57:04 -08002455
2456 if (flags & PAGE_WRITE) {
bellard9fa3e852004-01-04 18:06:42 +00002457 flags |= PAGE_WRITE_ORG;
Richard Henderson376a7902010-03-10 15:57:04 -08002458 }
2459
2460 for (addr = start, len = end - start;
2461 len != 0;
2462 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2463 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2464
2465 /* If the write protection bit is set, then we invalidate
2466 the code inside. */
ths5fafdf22007-09-16 21:08:06 +00002467 if (!(p->flags & PAGE_WRITE) &&
bellard9fa3e852004-01-04 18:06:42 +00002468 (flags & PAGE_WRITE) &&
2469 p->first_tb) {
bellardd720b932004-04-25 17:57:43 +00002470 tb_invalidate_phys_page(addr, 0, NULL);
bellard9fa3e852004-01-04 18:06:42 +00002471 }
2472 p->flags = flags;
2473 }
bellard9fa3e852004-01-04 18:06:42 +00002474}
2475
ths3d97b402007-11-02 19:02:07 +00002476int page_check_range(target_ulong start, target_ulong len, int flags)
2477{
2478 PageDesc *p;
2479 target_ulong end;
2480 target_ulong addr;
2481
Richard Henderson376a7902010-03-10 15:57:04 -08002482 /* This function should never be called with addresses outside the
2483 guest address space. If this assert fires, it probably indicates
2484 a missing call to h2g_valid. */
Blue Swirl338e9e62010-03-13 09:48:08 +00002485#if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2486 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
Richard Henderson376a7902010-03-10 15:57:04 -08002487#endif
2488
Richard Henderson3e0650a2010-03-29 10:54:42 -07002489 if (len == 0) {
2490 return 0;
2491 }
Richard Henderson376a7902010-03-10 15:57:04 -08002492 if (start + len - 1 < start) {
2493 /* We've wrapped around. */
balrog55f280c2008-10-28 10:24:11 +00002494 return -1;
Richard Henderson376a7902010-03-10 15:57:04 -08002495 }
balrog55f280c2008-10-28 10:24:11 +00002496
ths3d97b402007-11-02 19:02:07 +00002497 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2498 start = start & TARGET_PAGE_MASK;
2499
Richard Henderson376a7902010-03-10 15:57:04 -08002500 for (addr = start, len = end - start;
2501 len != 0;
2502 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
ths3d97b402007-11-02 19:02:07 +00002503 p = page_find(addr >> TARGET_PAGE_BITS);
2504 if( !p )
2505 return -1;
2506 if( !(p->flags & PAGE_VALID) )
2507 return -1;
2508
bellarddae32702007-11-14 10:51:00 +00002509 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
ths3d97b402007-11-02 19:02:07 +00002510 return -1;
bellarddae32702007-11-14 10:51:00 +00002511 if (flags & PAGE_WRITE) {
2512 if (!(p->flags & PAGE_WRITE_ORG))
2513 return -1;
2514 /* unprotect the page if it was put read-only because it
2515 contains translated code */
2516 if (!(p->flags & PAGE_WRITE)) {
2517 if (!page_unprotect(addr, 0, NULL))
2518 return -1;
2519 }
2520 return 0;
2521 }
ths3d97b402007-11-02 19:02:07 +00002522 }
2523 return 0;
2524}
2525
bellard9fa3e852004-01-04 18:06:42 +00002526/* called from signal handler: invalidate the code and unprotect the
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002527 page. Return TRUE if the fault was successfully handled. */
pbrook53a59602006-03-25 19:31:22 +00002528int page_unprotect(target_ulong address, unsigned long pc, void *puc)
bellard9fa3e852004-01-04 18:06:42 +00002529{
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002530 unsigned int prot;
2531 PageDesc *p;
pbrook53a59602006-03-25 19:31:22 +00002532 target_ulong host_start, host_end, addr;
bellard9fa3e852004-01-04 18:06:42 +00002533
pbrookc8a706f2008-06-02 16:16:42 +00002534 /* Technically this isn't safe inside a signal handler. However we
2535 know this only ever happens in a synchronous SEGV handler, so in
2536 practice it seems to be ok. */
2537 mmap_lock();
2538
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002539 p = page_find(address >> TARGET_PAGE_BITS);
2540 if (!p) {
pbrookc8a706f2008-06-02 16:16:42 +00002541 mmap_unlock();
bellard9fa3e852004-01-04 18:06:42 +00002542 return 0;
pbrookc8a706f2008-06-02 16:16:42 +00002543 }
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002544
bellard9fa3e852004-01-04 18:06:42 +00002545 /* if the page was really writable, then we change its
2546 protection back to writable */
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002547 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2548 host_start = address & qemu_host_page_mask;
2549 host_end = host_start + qemu_host_page_size;
2550
2551 prot = 0;
2552 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2553 p = page_find(addr >> TARGET_PAGE_BITS);
2554 p->flags |= PAGE_WRITE;
2555 prot |= p->flags;
2556
bellard9fa3e852004-01-04 18:06:42 +00002557 /* and since the content will be modified, we must invalidate
2558 the corresponding translated code. */
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002559 tb_invalidate_phys_page(addr, pc, puc);
bellard9fa3e852004-01-04 18:06:42 +00002560#ifdef DEBUG_TB_CHECK
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002561 tb_invalidate_check(addr);
bellard9fa3e852004-01-04 18:06:42 +00002562#endif
bellard9fa3e852004-01-04 18:06:42 +00002563 }
Aurelien Jarno45d679d2010-03-29 02:12:51 +02002564 mprotect((void *)g2h(host_start), qemu_host_page_size,
2565 prot & PAGE_BITS);
2566
2567 mmap_unlock();
2568 return 1;
bellard9fa3e852004-01-04 18:06:42 +00002569 }
pbrookc8a706f2008-06-02 16:16:42 +00002570 mmap_unlock();
bellard9fa3e852004-01-04 18:06:42 +00002571 return 0;
2572}
2573
bellard6a00d602005-11-21 23:25:50 +00002574static inline void tlb_set_dirty(CPUState *env,
2575 unsigned long addr, target_ulong vaddr)
bellard1ccde1c2004-02-06 19:46:14 +00002576{
2577}
bellard9fa3e852004-01-04 18:06:42 +00002578#endif /* defined(CONFIG_USER_ONLY) */
2579
pbrooke2eef172008-06-08 01:09:01 +00002580#if !defined(CONFIG_USER_ONLY)
pbrook8da3ff12008-12-01 18:59:50 +00002581
Paul Brookc04b2b72010-03-01 03:31:14 +00002582#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2583typedef struct subpage_t {
2584 target_phys_addr_t base;
Richard Hendersonf6405242010-04-22 16:47:31 -07002585 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2586 ram_addr_t region_offset[TARGET_PAGE_SIZE];
Paul Brookc04b2b72010-03-01 03:31:14 +00002587} subpage_t;
2588
Anthony Liguoric227f092009-10-01 16:12:16 -05002589static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2590 ram_addr_t memory, ram_addr_t region_offset);
Richard Hendersonf6405242010-04-22 16:47:31 -07002591static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2592 ram_addr_t orig_memory,
2593 ram_addr_t region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00002594#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2595 need_subpage) \
2596 do { \
2597 if (addr > start_addr) \
2598 start_addr2 = 0; \
2599 else { \
2600 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2601 if (start_addr2 > 0) \
2602 need_subpage = 1; \
2603 } \
2604 \
blueswir149e9fba2007-05-30 17:25:06 +00002605 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
blueswir1db7b5422007-05-26 17:36:03 +00002606 end_addr2 = TARGET_PAGE_SIZE - 1; \
2607 else { \
2608 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2609 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2610 need_subpage = 1; \
2611 } \
2612 } while (0)
2613
Michael S. Tsirkin8f2498f2009-09-29 18:53:16 +02002614/* register physical memory.
2615 For RAM, 'size' must be a multiple of the target page size.
2616 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
pbrook8da3ff12008-12-01 18:59:50 +00002617 io memory page. The address used when calling the IO function is
2618 the offset from the start of the region, plus region_offset. Both
Stuart Bradyccbb4d42009-05-03 12:15:06 +01002619 start_addr and region_offset are rounded down to a page boundary
pbrook8da3ff12008-12-01 18:59:50 +00002620 before calculating this offset. This should not be a problem unless
2621 the low bits of start_addr and region_offset differ. */
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002622void cpu_register_physical_memory_log(target_phys_addr_t start_addr,
Anthony Liguoric227f092009-10-01 16:12:16 -05002623 ram_addr_t size,
2624 ram_addr_t phys_offset,
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002625 ram_addr_t region_offset,
2626 bool log_dirty)
bellard33417e72003-08-10 21:47:01 +00002627{
Anthony Liguoric227f092009-10-01 16:12:16 -05002628 target_phys_addr_t addr, end_addr;
bellard92e873b2004-05-21 14:52:29 +00002629 PhysPageDesc *p;
bellard9d420372006-06-25 22:25:22 +00002630 CPUState *env;
Anthony Liguoric227f092009-10-01 16:12:16 -05002631 ram_addr_t orig_size = size;
Richard Hendersonf6405242010-04-22 16:47:31 -07002632 subpage_t *subpage;
bellard33417e72003-08-10 21:47:01 +00002633
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002634 assert(size);
Michael S. Tsirkin0fd542f2011-04-06 22:25:38 +03002635 cpu_notify_set_memory(start_addr, size, phys_offset, log_dirty);
Michael S. Tsirkinf6f3fbc2010-01-27 22:06:57 +02002636
pbrook67c4d232009-02-23 13:16:07 +00002637 if (phys_offset == IO_MEM_UNASSIGNED) {
2638 region_offset = start_addr;
2639 }
pbrook8da3ff12008-12-01 18:59:50 +00002640 region_offset &= TARGET_PAGE_MASK;
bellard5fd386f2004-05-23 21:11:22 +00002641 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
Anthony Liguoric227f092009-10-01 16:12:16 -05002642 end_addr = start_addr + (target_phys_addr_t)size;
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002643
2644 addr = start_addr;
2645 do {
blueswir1db7b5422007-05-26 17:36:03 +00002646 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2647 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
Anthony Liguoric227f092009-10-01 16:12:16 -05002648 ram_addr_t orig_memory = p->phys_offset;
2649 target_phys_addr_t start_addr2, end_addr2;
blueswir1db7b5422007-05-26 17:36:03 +00002650 int need_subpage = 0;
2651
2652 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2653 need_subpage);
Richard Hendersonf6405242010-04-22 16:47:31 -07002654 if (need_subpage) {
blueswir1db7b5422007-05-26 17:36:03 +00002655 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2656 subpage = subpage_init((addr & TARGET_PAGE_MASK),
pbrook8da3ff12008-12-01 18:59:50 +00002657 &p->phys_offset, orig_memory,
2658 p->region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00002659 } else {
2660 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2661 >> IO_MEM_SHIFT];
2662 }
pbrook8da3ff12008-12-01 18:59:50 +00002663 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2664 region_offset);
2665 p->region_offset = 0;
blueswir1db7b5422007-05-26 17:36:03 +00002666 } else {
2667 p->phys_offset = phys_offset;
2668 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2669 (phys_offset & IO_MEM_ROMD))
2670 phys_offset += TARGET_PAGE_SIZE;
2671 }
2672 } else {
2673 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2674 p->phys_offset = phys_offset;
pbrook8da3ff12008-12-01 18:59:50 +00002675 p->region_offset = region_offset;
blueswir1db7b5422007-05-26 17:36:03 +00002676 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
pbrook8da3ff12008-12-01 18:59:50 +00002677 (phys_offset & IO_MEM_ROMD)) {
blueswir1db7b5422007-05-26 17:36:03 +00002678 phys_offset += TARGET_PAGE_SIZE;
pbrook0e8f0962008-12-02 09:02:15 +00002679 } else {
Anthony Liguoric227f092009-10-01 16:12:16 -05002680 target_phys_addr_t start_addr2, end_addr2;
blueswir1db7b5422007-05-26 17:36:03 +00002681 int need_subpage = 0;
2682
2683 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2684 end_addr2, need_subpage);
2685
Richard Hendersonf6405242010-04-22 16:47:31 -07002686 if (need_subpage) {
blueswir1db7b5422007-05-26 17:36:03 +00002687 subpage = subpage_init((addr & TARGET_PAGE_MASK),
pbrook8da3ff12008-12-01 18:59:50 +00002688 &p->phys_offset, IO_MEM_UNASSIGNED,
pbrook67c4d232009-02-23 13:16:07 +00002689 addr & TARGET_PAGE_MASK);
blueswir1db7b5422007-05-26 17:36:03 +00002690 subpage_register(subpage, start_addr2, end_addr2,
pbrook8da3ff12008-12-01 18:59:50 +00002691 phys_offset, region_offset);
2692 p->region_offset = 0;
blueswir1db7b5422007-05-26 17:36:03 +00002693 }
2694 }
2695 }
pbrook8da3ff12008-12-01 18:59:50 +00002696 region_offset += TARGET_PAGE_SIZE;
Edgar E. Iglesias3b8e6a22011-04-05 13:00:36 +02002697 addr += TARGET_PAGE_SIZE;
2698 } while (addr != end_addr);
ths3b46e622007-09-17 08:09:54 +00002699
bellard9d420372006-06-25 22:25:22 +00002700 /* since each CPU stores ram addresses in its TLB cache, we must
2701 reset the modified entries */
2702 /* XXX: slow ! */
2703 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2704 tlb_flush(env, 1);
2705 }
bellard33417e72003-08-10 21:47:01 +00002706}
2707
bellardba863452006-09-24 18:41:10 +00002708/* XXX: temporary until new memory mapping API */
Anthony Liguoric227f092009-10-01 16:12:16 -05002709ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
bellardba863452006-09-24 18:41:10 +00002710{
2711 PhysPageDesc *p;
2712
2713 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2714 if (!p)
2715 return IO_MEM_UNASSIGNED;
2716 return p->phys_offset;
2717}
2718
Anthony Liguoric227f092009-10-01 16:12:16 -05002719void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
aliguorif65ed4c2008-12-09 20:09:57 +00002720{
2721 if (kvm_enabled())
2722 kvm_coalesce_mmio_region(addr, size);
2723}
2724
Anthony Liguoric227f092009-10-01 16:12:16 -05002725void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
aliguorif65ed4c2008-12-09 20:09:57 +00002726{
2727 if (kvm_enabled())
2728 kvm_uncoalesce_mmio_region(addr, size);
2729}
2730
Sheng Yang62a27442010-01-26 19:21:16 +08002731void qemu_flush_coalesced_mmio_buffer(void)
2732{
2733 if (kvm_enabled())
2734 kvm_flush_coalesced_mmio_buffer();
2735}
2736
Marcelo Tosattic9027602010-03-01 20:25:08 -03002737#if defined(__linux__) && !defined(TARGET_S390X)
2738
2739#include <sys/vfs.h>
2740
2741#define HUGETLBFS_MAGIC 0x958458f6
2742
2743static long gethugepagesize(const char *path)
2744{
2745 struct statfs fs;
2746 int ret;
2747
2748 do {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002749 ret = statfs(path, &fs);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002750 } while (ret != 0 && errno == EINTR);
2751
2752 if (ret != 0) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002753 perror(path);
2754 return 0;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002755 }
2756
2757 if (fs.f_type != HUGETLBFS_MAGIC)
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002758 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002759
2760 return fs.f_bsize;
2761}
2762
Alex Williamson04b16652010-07-02 11:13:17 -06002763static void *file_ram_alloc(RAMBlock *block,
2764 ram_addr_t memory,
2765 const char *path)
Marcelo Tosattic9027602010-03-01 20:25:08 -03002766{
2767 char *filename;
2768 void *area;
2769 int fd;
2770#ifdef MAP_POPULATE
2771 int flags;
2772#endif
2773 unsigned long hpagesize;
2774
2775 hpagesize = gethugepagesize(path);
2776 if (!hpagesize) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002777 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002778 }
2779
2780 if (memory < hpagesize) {
2781 return NULL;
2782 }
2783
2784 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2785 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2786 return NULL;
2787 }
2788
2789 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002790 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002791 }
2792
2793 fd = mkstemp(filename);
2794 if (fd < 0) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002795 perror("unable to create backing store for hugepages");
2796 free(filename);
2797 return NULL;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002798 }
2799 unlink(filename);
2800 free(filename);
2801
2802 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2803
2804 /*
2805 * ftruncate is not supported by hugetlbfs in older
2806 * hosts, so don't bother bailing out on errors.
2807 * If anything goes wrong with it under other filesystems,
2808 * mmap will fail.
2809 */
2810 if (ftruncate(fd, memory))
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002811 perror("ftruncate");
Marcelo Tosattic9027602010-03-01 20:25:08 -03002812
2813#ifdef MAP_POPULATE
2814 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2815 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2816 * to sidestep this quirk.
2817 */
2818 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2819 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2820#else
2821 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2822#endif
2823 if (area == MAP_FAILED) {
Yoshiaki Tamura9742bf22010-08-18 13:30:13 +09002824 perror("file_ram_alloc: can't mmap RAM pages");
2825 close(fd);
2826 return (NULL);
Marcelo Tosattic9027602010-03-01 20:25:08 -03002827 }
Alex Williamson04b16652010-07-02 11:13:17 -06002828 block->fd = fd;
Marcelo Tosattic9027602010-03-01 20:25:08 -03002829 return area;
2830}
2831#endif
2832
Alex Williamsond17b5282010-06-25 11:08:38 -06002833static ram_addr_t find_ram_offset(ram_addr_t size)
2834{
Alex Williamson04b16652010-07-02 11:13:17 -06002835 RAMBlock *block, *next_block;
Blue Swirl09d7ae92010-07-07 19:37:53 +00002836 ram_addr_t offset = 0, mingap = ULONG_MAX;
Alex Williamson04b16652010-07-02 11:13:17 -06002837
2838 if (QLIST_EMPTY(&ram_list.blocks))
2839 return 0;
2840
2841 QLIST_FOREACH(block, &ram_list.blocks, next) {
2842 ram_addr_t end, next = ULONG_MAX;
2843
2844 end = block->offset + block->length;
2845
2846 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2847 if (next_block->offset >= end) {
2848 next = MIN(next, next_block->offset);
2849 }
2850 }
2851 if (next - end >= size && next - end < mingap) {
2852 offset = end;
2853 mingap = next - end;
2854 }
2855 }
2856 return offset;
2857}
2858
2859static ram_addr_t last_ram_offset(void)
2860{
Alex Williamsond17b5282010-06-25 11:08:38 -06002861 RAMBlock *block;
2862 ram_addr_t last = 0;
2863
2864 QLIST_FOREACH(block, &ram_list.blocks, next)
2865 last = MAX(last, block->offset + block->length);
2866
2867 return last;
2868}
2869
Cam Macdonell84b89d72010-07-26 18:10:57 -06002870ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002871 ram_addr_t size, void *host)
Cam Macdonell84b89d72010-07-26 18:10:57 -06002872{
2873 RAMBlock *new_block, *block;
2874
2875 size = TARGET_PAGE_ALIGN(size);
2876 new_block = qemu_mallocz(sizeof(*new_block));
2877
2878 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2879 char *id = dev->parent_bus->info->get_dev_path(dev);
2880 if (id) {
2881 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2882 qemu_free(id);
2883 }
2884 }
2885 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2886
2887 QLIST_FOREACH(block, &ram_list.blocks, next) {
2888 if (!strcmp(block->idstr, new_block->idstr)) {
2889 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2890 new_block->idstr);
2891 abort();
2892 }
2893 }
2894
Jun Nakajima432d2682010-08-31 16:41:25 +01002895 new_block->offset = find_ram_offset(size);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002896 if (host) {
2897 new_block->host = host;
Huang Yingcd19cfa2011-03-02 08:56:19 +01002898 new_block->flags |= RAM_PREALLOC_MASK;
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002899 } else {
2900 if (mem_path) {
2901#if defined (__linux__) && !defined(TARGET_S390X)
2902 new_block->host = file_ram_alloc(new_block, size, mem_path);
2903 if (!new_block->host) {
2904 new_block->host = qemu_vmalloc(size);
Andreas Färbere78815a2010-09-25 11:26:05 +00002905 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002906 }
2907#else
2908 fprintf(stderr, "-mem-path option unsupported\n");
2909 exit(1);
2910#endif
2911 } else {
2912#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2913 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2914 new_block->host = mmap((void*)0x1000000, size,
2915 PROT_EXEC|PROT_READ|PROT_WRITE,
2916 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2917#else
Jun Nakajima432d2682010-08-31 16:41:25 +01002918 if (xen_mapcache_enabled()) {
2919 xen_ram_alloc(new_block->offset, size);
2920 } else {
2921 new_block->host = qemu_vmalloc(size);
2922 }
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002923#endif
Andreas Färbere78815a2010-09-25 11:26:05 +00002924 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002925 }
2926 }
Cam Macdonell84b89d72010-07-26 18:10:57 -06002927 new_block->length = size;
2928
2929 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2930
2931 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2932 last_ram_offset() >> TARGET_PAGE_BITS);
2933 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2934 0xff, size >> TARGET_PAGE_BITS);
2935
2936 if (kvm_enabled())
2937 kvm_setup_guest_memory(new_block->host, size);
2938
2939 return new_block->offset;
2940}
2941
Alex Williamson1724f042010-06-25 11:09:35 -06002942ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
pbrook94a6b542009-04-11 17:15:54 +00002943{
Yoshiaki Tamura6977dfe2010-08-18 15:41:49 +09002944 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
pbrook94a6b542009-04-11 17:15:54 +00002945}
bellarde9a1ab12007-02-08 23:08:38 +00002946
Anthony Liguoric227f092009-10-01 16:12:16 -05002947void qemu_ram_free(ram_addr_t addr)
bellarde9a1ab12007-02-08 23:08:38 +00002948{
Alex Williamson04b16652010-07-02 11:13:17 -06002949 RAMBlock *block;
2950
2951 QLIST_FOREACH(block, &ram_list.blocks, next) {
2952 if (addr == block->offset) {
2953 QLIST_REMOVE(block, next);
Huang Yingcd19cfa2011-03-02 08:56:19 +01002954 if (block->flags & RAM_PREALLOC_MASK) {
2955 ;
2956 } else if (mem_path) {
Alex Williamson04b16652010-07-02 11:13:17 -06002957#if defined (__linux__) && !defined(TARGET_S390X)
2958 if (block->fd) {
2959 munmap(block->host, block->length);
2960 close(block->fd);
2961 } else {
2962 qemu_vfree(block->host);
2963 }
Jan Kiszkafd28aa12011-03-15 12:26:14 +01002964#else
2965 abort();
Alex Williamson04b16652010-07-02 11:13:17 -06002966#endif
2967 } else {
2968#if defined(TARGET_S390X) && defined(CONFIG_KVM)
2969 munmap(block->host, block->length);
2970#else
Jun Nakajima432d2682010-08-31 16:41:25 +01002971 if (xen_mapcache_enabled()) {
2972 qemu_invalidate_entry(block->host);
2973 } else {
2974 qemu_vfree(block->host);
2975 }
Alex Williamson04b16652010-07-02 11:13:17 -06002976#endif
2977 }
2978 qemu_free(block);
2979 return;
2980 }
2981 }
2982
bellarde9a1ab12007-02-08 23:08:38 +00002983}
2984
Huang Yingcd19cfa2011-03-02 08:56:19 +01002985#ifndef _WIN32
2986void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2987{
2988 RAMBlock *block;
2989 ram_addr_t offset;
2990 int flags;
2991 void *area, *vaddr;
2992
2993 QLIST_FOREACH(block, &ram_list.blocks, next) {
2994 offset = addr - block->offset;
2995 if (offset < block->length) {
2996 vaddr = block->host + offset;
2997 if (block->flags & RAM_PREALLOC_MASK) {
2998 ;
2999 } else {
3000 flags = MAP_FIXED;
3001 munmap(vaddr, length);
3002 if (mem_path) {
3003#if defined(__linux__) && !defined(TARGET_S390X)
3004 if (block->fd) {
3005#ifdef MAP_POPULATE
3006 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
3007 MAP_PRIVATE;
3008#else
3009 flags |= MAP_PRIVATE;
3010#endif
3011 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3012 flags, block->fd, offset);
3013 } else {
3014 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3015 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3016 flags, -1, 0);
3017 }
Jan Kiszkafd28aa12011-03-15 12:26:14 +01003018#else
3019 abort();
Huang Yingcd19cfa2011-03-02 08:56:19 +01003020#endif
3021 } else {
3022#if defined(TARGET_S390X) && defined(CONFIG_KVM)
3023 flags |= MAP_SHARED | MAP_ANONYMOUS;
3024 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
3025 flags, -1, 0);
3026#else
3027 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3028 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3029 flags, -1, 0);
3030#endif
3031 }
3032 if (area != vaddr) {
3033 fprintf(stderr, "Could not remap addr: %lx@%lx\n",
3034 length, addr);
3035 exit(1);
3036 }
3037 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3038 }
3039 return;
3040 }
3041 }
3042}
3043#endif /* !_WIN32 */
3044
pbrookdc828ca2009-04-09 22:21:07 +00003045/* Return a host pointer to ram allocated with qemu_ram_alloc.
pbrook5579c7f2009-04-11 14:47:08 +00003046 With the exception of the softmmu code in this file, this should
3047 only be used for local memory (e.g. video ram) that the device owns,
3048 and knows it isn't going to access beyond the end of the block.
3049
3050 It should not be used for general purpose DMA.
3051 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3052 */
Anthony Liguoric227f092009-10-01 16:12:16 -05003053void *qemu_get_ram_ptr(ram_addr_t addr)
pbrookdc828ca2009-04-09 22:21:07 +00003054{
pbrook94a6b542009-04-11 17:15:54 +00003055 RAMBlock *block;
3056
Alex Williamsonf471a172010-06-11 11:11:42 -06003057 QLIST_FOREACH(block, &ram_list.blocks, next) {
3058 if (addr - block->offset < block->length) {
Vincent Palatin7d82af32011-03-10 15:47:46 -05003059 /* Move this entry to to start of the list. */
3060 if (block != QLIST_FIRST(&ram_list.blocks)) {
3061 QLIST_REMOVE(block, next);
3062 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3063 }
Jun Nakajima432d2682010-08-31 16:41:25 +01003064 if (xen_mapcache_enabled()) {
3065 /* We need to check if the requested address is in the RAM
3066 * because we don't want to map the entire memory in QEMU.
3067 */
3068 if (block->offset == 0) {
3069 return qemu_map_cache(addr, 0, 1);
3070 } else if (block->host == NULL) {
3071 block->host = xen_map_block(block->offset, block->length);
3072 }
3073 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003074 return block->host + (addr - block->offset);
3075 }
pbrook94a6b542009-04-11 17:15:54 +00003076 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003077
3078 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3079 abort();
3080
3081 return NULL;
pbrookdc828ca2009-04-09 22:21:07 +00003082}
3083
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02003084/* Return a host pointer to ram allocated with qemu_ram_alloc.
3085 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3086 */
3087void *qemu_safe_ram_ptr(ram_addr_t addr)
3088{
3089 RAMBlock *block;
3090
3091 QLIST_FOREACH(block, &ram_list.blocks, next) {
3092 if (addr - block->offset < block->length) {
Jun Nakajima432d2682010-08-31 16:41:25 +01003093 if (xen_mapcache_enabled()) {
3094 /* We need to check if the requested address is in the RAM
3095 * because we don't want to map the entire memory in QEMU.
3096 */
3097 if (block->offset == 0) {
3098 return qemu_map_cache(addr, 0, 1);
3099 } else if (block->host == NULL) {
3100 block->host = xen_map_block(block->offset, block->length);
3101 }
3102 }
Michael S. Tsirkinb2e0a132010-11-22 19:52:34 +02003103 return block->host + (addr - block->offset);
3104 }
3105 }
3106
3107 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3108 abort();
3109
3110 return NULL;
3111}
3112
Marcelo Tosattie8902612010-10-11 15:31:19 -03003113int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
pbrook5579c7f2009-04-11 14:47:08 +00003114{
pbrook94a6b542009-04-11 17:15:54 +00003115 RAMBlock *block;
3116 uint8_t *host = ptr;
3117
Alex Williamsonf471a172010-06-11 11:11:42 -06003118 QLIST_FOREACH(block, &ram_list.blocks, next) {
Jun Nakajima432d2682010-08-31 16:41:25 +01003119 /* This case append when the block is not mapped. */
3120 if (block->host == NULL) {
3121 continue;
3122 }
Alex Williamsonf471a172010-06-11 11:11:42 -06003123 if (host - block->host < block->length) {
Marcelo Tosattie8902612010-10-11 15:31:19 -03003124 *ram_addr = block->offset + (host - block->host);
3125 return 0;
Alex Williamsonf471a172010-06-11 11:11:42 -06003126 }
pbrook94a6b542009-04-11 17:15:54 +00003127 }
Jun Nakajima432d2682010-08-31 16:41:25 +01003128
3129 if (xen_mapcache_enabled()) {
3130 *ram_addr = qemu_ram_addr_from_mapcache(ptr);
3131 return 0;
3132 }
3133
Marcelo Tosattie8902612010-10-11 15:31:19 -03003134 return -1;
3135}
Alex Williamsonf471a172010-06-11 11:11:42 -06003136
Marcelo Tosattie8902612010-10-11 15:31:19 -03003137/* Some of the softmmu routines need to translate from a host pointer
3138 (typically a TLB entry) back to a ram offset. */
3139ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3140{
3141 ram_addr_t ram_addr;
Alex Williamsonf471a172010-06-11 11:11:42 -06003142
Marcelo Tosattie8902612010-10-11 15:31:19 -03003143 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3144 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3145 abort();
3146 }
3147 return ram_addr;
pbrook5579c7f2009-04-11 14:47:08 +00003148}
3149
Anthony Liguoric227f092009-10-01 16:12:16 -05003150static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
bellard33417e72003-08-10 21:47:01 +00003151{
pbrook67d3b952006-12-18 05:03:52 +00003152#ifdef DEBUG_UNASSIGNED
blueswir1ab3d1722007-11-04 07:31:40 +00003153 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
pbrook67d3b952006-12-18 05:03:52 +00003154#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003155#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003156 do_unassigned_access(addr, 0, 0, 0, 1);
3157#endif
3158 return 0;
3159}
3160
Anthony Liguoric227f092009-10-01 16:12:16 -05003161static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
blueswir1e18231a2008-10-06 18:46:28 +00003162{
3163#ifdef DEBUG_UNASSIGNED
3164 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3165#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003166#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003167 do_unassigned_access(addr, 0, 0, 0, 2);
3168#endif
3169 return 0;
3170}
3171
Anthony Liguoric227f092009-10-01 16:12:16 -05003172static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
blueswir1e18231a2008-10-06 18:46:28 +00003173{
3174#ifdef DEBUG_UNASSIGNED
3175 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3176#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003177#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003178 do_unassigned_access(addr, 0, 0, 0, 4);
blueswir1b4f0a312007-05-06 17:59:24 +00003179#endif
bellard33417e72003-08-10 21:47:01 +00003180 return 0;
3181}
3182
Anthony Liguoric227f092009-10-01 16:12:16 -05003183static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
bellard33417e72003-08-10 21:47:01 +00003184{
pbrook67d3b952006-12-18 05:03:52 +00003185#ifdef DEBUG_UNASSIGNED
blueswir1ab3d1722007-11-04 07:31:40 +00003186 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
pbrook67d3b952006-12-18 05:03:52 +00003187#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003188#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003189 do_unassigned_access(addr, 1, 0, 0, 1);
3190#endif
3191}
3192
Anthony Liguoric227f092009-10-01 16:12:16 -05003193static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
blueswir1e18231a2008-10-06 18:46:28 +00003194{
3195#ifdef DEBUG_UNASSIGNED
3196 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3197#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003198#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003199 do_unassigned_access(addr, 1, 0, 0, 2);
3200#endif
3201}
3202
Anthony Liguoric227f092009-10-01 16:12:16 -05003203static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
blueswir1e18231a2008-10-06 18:46:28 +00003204{
3205#ifdef DEBUG_UNASSIGNED
3206 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3207#endif
Edgar E. Iglesiasfaed1c22009-09-03 13:25:09 +02003208#if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
blueswir1e18231a2008-10-06 18:46:28 +00003209 do_unassigned_access(addr, 1, 0, 0, 4);
blueswir1b4f0a312007-05-06 17:59:24 +00003210#endif
bellard33417e72003-08-10 21:47:01 +00003211}
3212
Blue Swirld60efc62009-08-25 18:29:31 +00003213static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
bellard33417e72003-08-10 21:47:01 +00003214 unassigned_mem_readb,
blueswir1e18231a2008-10-06 18:46:28 +00003215 unassigned_mem_readw,
3216 unassigned_mem_readl,
bellard33417e72003-08-10 21:47:01 +00003217};
3218
Blue Swirld60efc62009-08-25 18:29:31 +00003219static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
bellard33417e72003-08-10 21:47:01 +00003220 unassigned_mem_writeb,
blueswir1e18231a2008-10-06 18:46:28 +00003221 unassigned_mem_writew,
3222 unassigned_mem_writel,
bellard33417e72003-08-10 21:47:01 +00003223};
3224
Anthony Liguoric227f092009-10-01 16:12:16 -05003225static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003226 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003227{
bellard3a7d9292005-08-21 09:26:42 +00003228 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003229 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003230 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3231#if !defined(CONFIG_USER_ONLY)
3232 tb_invalidate_phys_page_fast(ram_addr, 1);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003233 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003234#endif
3235 }
pbrook5579c7f2009-04-11 14:47:08 +00003236 stb_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003237 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003238 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003239 /* we remove the notdirty callback only if the code has been
3240 flushed */
3241 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003242 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003243}
3244
Anthony Liguoric227f092009-10-01 16:12:16 -05003245static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003246 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003247{
bellard3a7d9292005-08-21 09:26:42 +00003248 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003249 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003250 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3251#if !defined(CONFIG_USER_ONLY)
3252 tb_invalidate_phys_page_fast(ram_addr, 2);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003253 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003254#endif
3255 }
pbrook5579c7f2009-04-11 14:47:08 +00003256 stw_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003257 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003258 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003259 /* we remove the notdirty callback only if the code has been
3260 flushed */
3261 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003262 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003263}
3264
Anthony Liguoric227f092009-10-01 16:12:16 -05003265static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
pbrook0f459d12008-06-09 00:20:13 +00003266 uint32_t val)
bellard1ccde1c2004-02-06 19:46:14 +00003267{
bellard3a7d9292005-08-21 09:26:42 +00003268 int dirty_flags;
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003269 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003270 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3271#if !defined(CONFIG_USER_ONLY)
3272 tb_invalidate_phys_page_fast(ram_addr, 4);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003273 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
bellard3a7d9292005-08-21 09:26:42 +00003274#endif
3275 }
pbrook5579c7f2009-04-11 14:47:08 +00003276 stl_p(qemu_get_ram_ptr(ram_addr), val);
bellardf23db162005-08-21 19:12:28 +00003277 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003278 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
bellardf23db162005-08-21 19:12:28 +00003279 /* we remove the notdirty callback only if the code has been
3280 flushed */
3281 if (dirty_flags == 0xff)
pbrook2e70f6e2008-06-29 01:03:05 +00003282 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
bellard1ccde1c2004-02-06 19:46:14 +00003283}
3284
Blue Swirld60efc62009-08-25 18:29:31 +00003285static CPUReadMemoryFunc * const error_mem_read[3] = {
bellard3a7d9292005-08-21 09:26:42 +00003286 NULL, /* never used */
3287 NULL, /* never used */
3288 NULL, /* never used */
3289};
3290
Blue Swirld60efc62009-08-25 18:29:31 +00003291static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
bellard1ccde1c2004-02-06 19:46:14 +00003292 notdirty_mem_writeb,
3293 notdirty_mem_writew,
3294 notdirty_mem_writel,
3295};
3296
pbrook0f459d12008-06-09 00:20:13 +00003297/* Generate a debug exception if a watchpoint has been hit. */
aliguorib4051332008-11-18 20:14:20 +00003298static void check_watchpoint(int offset, int len_mask, int flags)
pbrook0f459d12008-06-09 00:20:13 +00003299{
3300 CPUState *env = cpu_single_env;
aliguori06d55cc2008-11-18 20:24:06 +00003301 target_ulong pc, cs_base;
3302 TranslationBlock *tb;
pbrook0f459d12008-06-09 00:20:13 +00003303 target_ulong vaddr;
aliguoria1d1bb32008-11-18 20:07:32 +00003304 CPUWatchpoint *wp;
aliguori06d55cc2008-11-18 20:24:06 +00003305 int cpu_flags;
pbrook0f459d12008-06-09 00:20:13 +00003306
aliguori06d55cc2008-11-18 20:24:06 +00003307 if (env->watchpoint_hit) {
3308 /* We re-entered the check after replacing the TB. Now raise
3309 * the debug interrupt so that is will trigger after the
3310 * current instruction. */
3311 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3312 return;
3313 }
pbrook2e70f6e2008-06-29 01:03:05 +00003314 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
Blue Swirl72cf2d42009-09-12 07:36:22 +00003315 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
aliguorib4051332008-11-18 20:14:20 +00003316 if ((vaddr == (wp->vaddr & len_mask) ||
3317 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
aliguori6e140f22008-11-18 20:37:55 +00003318 wp->flags |= BP_WATCHPOINT_HIT;
3319 if (!env->watchpoint_hit) {
3320 env->watchpoint_hit = wp;
3321 tb = tb_find_pc(env->mem_io_pc);
3322 if (!tb) {
3323 cpu_abort(env, "check_watchpoint: could not find TB for "
3324 "pc=%p", (void *)env->mem_io_pc);
3325 }
Stefan Weil618ba8e2011-04-18 06:39:53 +00003326 cpu_restore_state(tb, env, env->mem_io_pc);
aliguori6e140f22008-11-18 20:37:55 +00003327 tb_phys_invalidate(tb, -1);
3328 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3329 env->exception_index = EXCP_DEBUG;
3330 } else {
3331 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3332 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3333 }
3334 cpu_resume_from_signal(env, NULL);
aliguori06d55cc2008-11-18 20:24:06 +00003335 }
aliguori6e140f22008-11-18 20:37:55 +00003336 } else {
3337 wp->flags &= ~BP_WATCHPOINT_HIT;
pbrook0f459d12008-06-09 00:20:13 +00003338 }
3339 }
3340}
3341
pbrook6658ffb2007-03-16 23:58:11 +00003342/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3343 so these check for a hit then pass through to the normal out-of-line
3344 phys routines. */
Anthony Liguoric227f092009-10-01 16:12:16 -05003345static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003346{
aliguorib4051332008-11-18 20:14:20 +00003347 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003348 return ldub_phys(addr);
3349}
3350
Anthony Liguoric227f092009-10-01 16:12:16 -05003351static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003352{
aliguorib4051332008-11-18 20:14:20 +00003353 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003354 return lduw_phys(addr);
3355}
3356
Anthony Liguoric227f092009-10-01 16:12:16 -05003357static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
pbrook6658ffb2007-03-16 23:58:11 +00003358{
aliguorib4051332008-11-18 20:14:20 +00003359 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
pbrook6658ffb2007-03-16 23:58:11 +00003360 return ldl_phys(addr);
3361}
3362
Anthony Liguoric227f092009-10-01 16:12:16 -05003363static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003364 uint32_t val)
3365{
aliguorib4051332008-11-18 20:14:20 +00003366 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003367 stb_phys(addr, val);
3368}
3369
Anthony Liguoric227f092009-10-01 16:12:16 -05003370static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003371 uint32_t val)
3372{
aliguorib4051332008-11-18 20:14:20 +00003373 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003374 stw_phys(addr, val);
3375}
3376
Anthony Liguoric227f092009-10-01 16:12:16 -05003377static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
pbrook6658ffb2007-03-16 23:58:11 +00003378 uint32_t val)
3379{
aliguorib4051332008-11-18 20:14:20 +00003380 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
pbrook6658ffb2007-03-16 23:58:11 +00003381 stl_phys(addr, val);
3382}
3383
Blue Swirld60efc62009-08-25 18:29:31 +00003384static CPUReadMemoryFunc * const watch_mem_read[3] = {
pbrook6658ffb2007-03-16 23:58:11 +00003385 watch_mem_readb,
3386 watch_mem_readw,
3387 watch_mem_readl,
3388};
3389
Blue Swirld60efc62009-08-25 18:29:31 +00003390static CPUWriteMemoryFunc * const watch_mem_write[3] = {
pbrook6658ffb2007-03-16 23:58:11 +00003391 watch_mem_writeb,
3392 watch_mem_writew,
3393 watch_mem_writel,
3394};
pbrook6658ffb2007-03-16 23:58:11 +00003395
Richard Hendersonf6405242010-04-22 16:47:31 -07003396static inline uint32_t subpage_readlen (subpage_t *mmio,
3397 target_phys_addr_t addr,
3398 unsigned int len)
blueswir1db7b5422007-05-26 17:36:03 +00003399{
Richard Hendersonf6405242010-04-22 16:47:31 -07003400 unsigned int idx = SUBPAGE_IDX(addr);
blueswir1db7b5422007-05-26 17:36:03 +00003401#if defined(DEBUG_SUBPAGE)
3402 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3403 mmio, len, addr, idx);
3404#endif
blueswir1db7b5422007-05-26 17:36:03 +00003405
Richard Hendersonf6405242010-04-22 16:47:31 -07003406 addr += mmio->region_offset[idx];
3407 idx = mmio->sub_io_index[idx];
3408 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
blueswir1db7b5422007-05-26 17:36:03 +00003409}
3410
Anthony Liguoric227f092009-10-01 16:12:16 -05003411static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
Richard Hendersonf6405242010-04-22 16:47:31 -07003412 uint32_t value, unsigned int len)
blueswir1db7b5422007-05-26 17:36:03 +00003413{
Richard Hendersonf6405242010-04-22 16:47:31 -07003414 unsigned int idx = SUBPAGE_IDX(addr);
blueswir1db7b5422007-05-26 17:36:03 +00003415#if defined(DEBUG_SUBPAGE)
Richard Hendersonf6405242010-04-22 16:47:31 -07003416 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3417 __func__, mmio, len, addr, idx, value);
blueswir1db7b5422007-05-26 17:36:03 +00003418#endif
Richard Hendersonf6405242010-04-22 16:47:31 -07003419
3420 addr += mmio->region_offset[idx];
3421 idx = mmio->sub_io_index[idx];
3422 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
blueswir1db7b5422007-05-26 17:36:03 +00003423}
3424
Anthony Liguoric227f092009-10-01 16:12:16 -05003425static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003426{
blueswir1db7b5422007-05-26 17:36:03 +00003427 return subpage_readlen(opaque, addr, 0);
3428}
3429
Anthony Liguoric227f092009-10-01 16:12:16 -05003430static void subpage_writeb (void *opaque, target_phys_addr_t addr,
blueswir1db7b5422007-05-26 17:36:03 +00003431 uint32_t value)
3432{
blueswir1db7b5422007-05-26 17:36:03 +00003433 subpage_writelen(opaque, addr, value, 0);
3434}
3435
Anthony Liguoric227f092009-10-01 16:12:16 -05003436static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003437{
blueswir1db7b5422007-05-26 17:36:03 +00003438 return subpage_readlen(opaque, addr, 1);
3439}
3440
Anthony Liguoric227f092009-10-01 16:12:16 -05003441static void subpage_writew (void *opaque, target_phys_addr_t addr,
blueswir1db7b5422007-05-26 17:36:03 +00003442 uint32_t value)
3443{
blueswir1db7b5422007-05-26 17:36:03 +00003444 subpage_writelen(opaque, addr, value, 1);
3445}
3446
Anthony Liguoric227f092009-10-01 16:12:16 -05003447static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
blueswir1db7b5422007-05-26 17:36:03 +00003448{
blueswir1db7b5422007-05-26 17:36:03 +00003449 return subpage_readlen(opaque, addr, 2);
3450}
3451
Richard Hendersonf6405242010-04-22 16:47:31 -07003452static void subpage_writel (void *opaque, target_phys_addr_t addr,
3453 uint32_t value)
blueswir1db7b5422007-05-26 17:36:03 +00003454{
blueswir1db7b5422007-05-26 17:36:03 +00003455 subpage_writelen(opaque, addr, value, 2);
3456}
3457
Blue Swirld60efc62009-08-25 18:29:31 +00003458static CPUReadMemoryFunc * const subpage_read[] = {
blueswir1db7b5422007-05-26 17:36:03 +00003459 &subpage_readb,
3460 &subpage_readw,
3461 &subpage_readl,
3462};
3463
Blue Swirld60efc62009-08-25 18:29:31 +00003464static CPUWriteMemoryFunc * const subpage_write[] = {
blueswir1db7b5422007-05-26 17:36:03 +00003465 &subpage_writeb,
3466 &subpage_writew,
3467 &subpage_writel,
3468};
3469
Anthony Liguoric227f092009-10-01 16:12:16 -05003470static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3471 ram_addr_t memory, ram_addr_t region_offset)
blueswir1db7b5422007-05-26 17:36:03 +00003472{
3473 int idx, eidx;
3474
3475 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3476 return -1;
3477 idx = SUBPAGE_IDX(start);
3478 eidx = SUBPAGE_IDX(end);
3479#if defined(DEBUG_SUBPAGE)
Blue Swirl0bf9e312009-07-20 17:19:25 +00003480 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
blueswir1db7b5422007-05-26 17:36:03 +00003481 mmio, start, end, idx, eidx, memory);
3482#endif
Gleb Natapov95c318f2010-07-29 10:41:45 +03003483 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3484 memory = IO_MEM_UNASSIGNED;
Richard Hendersonf6405242010-04-22 16:47:31 -07003485 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
blueswir1db7b5422007-05-26 17:36:03 +00003486 for (; idx <= eidx; idx++) {
Richard Hendersonf6405242010-04-22 16:47:31 -07003487 mmio->sub_io_index[idx] = memory;
3488 mmio->region_offset[idx] = region_offset;
blueswir1db7b5422007-05-26 17:36:03 +00003489 }
3490
3491 return 0;
3492}
3493
Richard Hendersonf6405242010-04-22 16:47:31 -07003494static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3495 ram_addr_t orig_memory,
3496 ram_addr_t region_offset)
blueswir1db7b5422007-05-26 17:36:03 +00003497{
Anthony Liguoric227f092009-10-01 16:12:16 -05003498 subpage_t *mmio;
blueswir1db7b5422007-05-26 17:36:03 +00003499 int subpage_memory;
3500
Anthony Liguoric227f092009-10-01 16:12:16 -05003501 mmio = qemu_mallocz(sizeof(subpage_t));
aliguori1eec6142009-02-05 22:06:18 +00003502
3503 mmio->base = base;
Alexander Graf2507c122010-12-08 12:05:37 +01003504 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3505 DEVICE_NATIVE_ENDIAN);
blueswir1db7b5422007-05-26 17:36:03 +00003506#if defined(DEBUG_SUBPAGE)
aliguori1eec6142009-02-05 22:06:18 +00003507 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3508 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
blueswir1db7b5422007-05-26 17:36:03 +00003509#endif
aliguori1eec6142009-02-05 22:06:18 +00003510 *phys = subpage_memory | IO_MEM_SUBPAGE;
Richard Hendersonf6405242010-04-22 16:47:31 -07003511 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
blueswir1db7b5422007-05-26 17:36:03 +00003512
3513 return mmio;
3514}
3515
aliguori88715652009-02-11 15:20:58 +00003516static int get_free_io_mem_idx(void)
3517{
3518 int i;
3519
3520 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3521 if (!io_mem_used[i]) {
3522 io_mem_used[i] = 1;
3523 return i;
3524 }
Riku Voipioc6703b42009-12-03 15:56:05 +02003525 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
aliguori88715652009-02-11 15:20:58 +00003526 return -1;
3527}
3528
Alexander Grafdd310532010-12-08 12:05:36 +01003529/*
3530 * Usually, devices operate in little endian mode. There are devices out
3531 * there that operate in big endian too. Each device gets byte swapped
3532 * mmio if plugged onto a CPU that does the other endianness.
3533 *
3534 * CPU Device swap?
3535 *
3536 * little little no
3537 * little big yes
3538 * big little yes
3539 * big big no
3540 */
3541
3542typedef struct SwapEndianContainer {
3543 CPUReadMemoryFunc *read[3];
3544 CPUWriteMemoryFunc *write[3];
3545 void *opaque;
3546} SwapEndianContainer;
3547
3548static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3549{
3550 uint32_t val;
3551 SwapEndianContainer *c = opaque;
3552 val = c->read[0](c->opaque, addr);
3553 return val;
3554}
3555
3556static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3557{
3558 uint32_t val;
3559 SwapEndianContainer *c = opaque;
3560 val = bswap16(c->read[1](c->opaque, addr));
3561 return val;
3562}
3563
3564static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3565{
3566 uint32_t val;
3567 SwapEndianContainer *c = opaque;
3568 val = bswap32(c->read[2](c->opaque, addr));
3569 return val;
3570}
3571
3572static CPUReadMemoryFunc * const swapendian_readfn[3]={
3573 swapendian_mem_readb,
3574 swapendian_mem_readw,
3575 swapendian_mem_readl
3576};
3577
3578static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3579 uint32_t val)
3580{
3581 SwapEndianContainer *c = opaque;
3582 c->write[0](c->opaque, addr, val);
3583}
3584
3585static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3586 uint32_t val)
3587{
3588 SwapEndianContainer *c = opaque;
3589 c->write[1](c->opaque, addr, bswap16(val));
3590}
3591
3592static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3593 uint32_t val)
3594{
3595 SwapEndianContainer *c = opaque;
3596 c->write[2](c->opaque, addr, bswap32(val));
3597}
3598
3599static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3600 swapendian_mem_writeb,
3601 swapendian_mem_writew,
3602 swapendian_mem_writel
3603};
3604
3605static void swapendian_init(int io_index)
3606{
3607 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3608 int i;
3609
3610 /* Swap mmio for big endian targets */
3611 c->opaque = io_mem_opaque[io_index];
3612 for (i = 0; i < 3; i++) {
3613 c->read[i] = io_mem_read[io_index][i];
3614 c->write[i] = io_mem_write[io_index][i];
3615
3616 io_mem_read[io_index][i] = swapendian_readfn[i];
3617 io_mem_write[io_index][i] = swapendian_writefn[i];
3618 }
3619 io_mem_opaque[io_index] = c;
3620}
3621
3622static void swapendian_del(int io_index)
3623{
3624 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3625 qemu_free(io_mem_opaque[io_index]);
3626 }
3627}
3628
bellard33417e72003-08-10 21:47:01 +00003629/* mem_read and mem_write are arrays of functions containing the
3630 function to access byte (index 0), word (index 1) and dword (index
Paul Brook0b4e6e32009-04-30 18:37:55 +01003631 2). Functions can be omitted with a NULL function pointer.
blueswir13ee89922008-01-02 19:45:26 +00003632 If io_index is non zero, the corresponding io zone is
blueswir14254fab2008-01-01 16:57:19 +00003633 modified. If it is zero, a new io zone is allocated. The return
3634 value can be used with cpu_register_physical_memory(). (-1) is
3635 returned if error. */
Avi Kivity1eed09c2009-06-14 11:38:51 +03003636static int cpu_register_io_memory_fixed(int io_index,
Blue Swirld60efc62009-08-25 18:29:31 +00003637 CPUReadMemoryFunc * const *mem_read,
3638 CPUWriteMemoryFunc * const *mem_write,
Alexander Grafdd310532010-12-08 12:05:36 +01003639 void *opaque, enum device_endian endian)
bellard33417e72003-08-10 21:47:01 +00003640{
Richard Henderson3cab7212010-05-07 09:52:51 -07003641 int i;
3642
bellard33417e72003-08-10 21:47:01 +00003643 if (io_index <= 0) {
aliguori88715652009-02-11 15:20:58 +00003644 io_index = get_free_io_mem_idx();
3645 if (io_index == -1)
3646 return io_index;
bellard33417e72003-08-10 21:47:01 +00003647 } else {
Avi Kivity1eed09c2009-06-14 11:38:51 +03003648 io_index >>= IO_MEM_SHIFT;
bellard33417e72003-08-10 21:47:01 +00003649 if (io_index >= IO_MEM_NB_ENTRIES)
3650 return -1;
3651 }
bellardb5ff1b32005-11-26 10:38:39 +00003652
Richard Henderson3cab7212010-05-07 09:52:51 -07003653 for (i = 0; i < 3; ++i) {
3654 io_mem_read[io_index][i]
3655 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3656 }
3657 for (i = 0; i < 3; ++i) {
3658 io_mem_write[io_index][i]
3659 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3660 }
bellarda4193c82004-06-03 14:01:43 +00003661 io_mem_opaque[io_index] = opaque;
Richard Hendersonf6405242010-04-22 16:47:31 -07003662
Alexander Grafdd310532010-12-08 12:05:36 +01003663 switch (endian) {
3664 case DEVICE_BIG_ENDIAN:
3665#ifndef TARGET_WORDS_BIGENDIAN
3666 swapendian_init(io_index);
3667#endif
3668 break;
3669 case DEVICE_LITTLE_ENDIAN:
3670#ifdef TARGET_WORDS_BIGENDIAN
3671 swapendian_init(io_index);
3672#endif
3673 break;
3674 case DEVICE_NATIVE_ENDIAN:
3675 default:
3676 break;
3677 }
3678
Richard Hendersonf6405242010-04-22 16:47:31 -07003679 return (io_index << IO_MEM_SHIFT);
bellard33417e72003-08-10 21:47:01 +00003680}
bellard61382a52003-10-27 21:22:23 +00003681
Blue Swirld60efc62009-08-25 18:29:31 +00003682int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3683 CPUWriteMemoryFunc * const *mem_write,
Alexander Grafdd310532010-12-08 12:05:36 +01003684 void *opaque, enum device_endian endian)
Avi Kivity1eed09c2009-06-14 11:38:51 +03003685{
Alexander Graf2507c122010-12-08 12:05:37 +01003686 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
Avi Kivity1eed09c2009-06-14 11:38:51 +03003687}
3688
aliguori88715652009-02-11 15:20:58 +00003689void cpu_unregister_io_memory(int io_table_address)
3690{
3691 int i;
3692 int io_index = io_table_address >> IO_MEM_SHIFT;
3693
Alexander Grafdd310532010-12-08 12:05:36 +01003694 swapendian_del(io_index);
3695
aliguori88715652009-02-11 15:20:58 +00003696 for (i=0;i < 3; i++) {
3697 io_mem_read[io_index][i] = unassigned_mem_read[i];
3698 io_mem_write[io_index][i] = unassigned_mem_write[i];
3699 }
3700 io_mem_opaque[io_index] = NULL;
3701 io_mem_used[io_index] = 0;
3702}
3703
Avi Kivitye9179ce2009-06-14 11:38:52 +03003704static void io_mem_init(void)
3705{
3706 int i;
3707
Alexander Graf2507c122010-12-08 12:05:37 +01003708 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3709 unassigned_mem_write, NULL,
3710 DEVICE_NATIVE_ENDIAN);
3711 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3712 unassigned_mem_write, NULL,
3713 DEVICE_NATIVE_ENDIAN);
3714 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3715 notdirty_mem_write, NULL,
3716 DEVICE_NATIVE_ENDIAN);
Avi Kivitye9179ce2009-06-14 11:38:52 +03003717 for (i=0; i<5; i++)
3718 io_mem_used[i] = 1;
3719
3720 io_mem_watch = cpu_register_io_memory(watch_mem_read,
Alexander Graf2507c122010-12-08 12:05:37 +01003721 watch_mem_write, NULL,
3722 DEVICE_NATIVE_ENDIAN);
Avi Kivitye9179ce2009-06-14 11:38:52 +03003723}
3724
pbrooke2eef172008-06-08 01:09:01 +00003725#endif /* !defined(CONFIG_USER_ONLY) */
3726
bellard13eb76e2004-01-24 15:23:36 +00003727/* physical memory access (slow version, mainly for debug) */
3728#if defined(CONFIG_USER_ONLY)
Paul Brooka68fe892010-03-01 00:08:59 +00003729int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3730 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00003731{
3732 int l, flags;
3733 target_ulong page;
pbrook53a59602006-03-25 19:31:22 +00003734 void * p;
bellard13eb76e2004-01-24 15:23:36 +00003735
3736 while (len > 0) {
3737 page = addr & TARGET_PAGE_MASK;
3738 l = (page + TARGET_PAGE_SIZE) - addr;
3739 if (l > len)
3740 l = len;
3741 flags = page_get_flags(page);
3742 if (!(flags & PAGE_VALID))
Paul Brooka68fe892010-03-01 00:08:59 +00003743 return -1;
bellard13eb76e2004-01-24 15:23:36 +00003744 if (is_write) {
3745 if (!(flags & PAGE_WRITE))
Paul Brooka68fe892010-03-01 00:08:59 +00003746 return -1;
bellard579a97f2007-11-11 14:26:47 +00003747 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00003748 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
Paul Brooka68fe892010-03-01 00:08:59 +00003749 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00003750 memcpy(p, buf, l);
3751 unlock_user(p, addr, l);
bellard13eb76e2004-01-24 15:23:36 +00003752 } else {
3753 if (!(flags & PAGE_READ))
Paul Brooka68fe892010-03-01 00:08:59 +00003754 return -1;
bellard579a97f2007-11-11 14:26:47 +00003755 /* XXX: this code should not depend on lock_user */
aurel3272fb7da2008-04-27 23:53:45 +00003756 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
Paul Brooka68fe892010-03-01 00:08:59 +00003757 return -1;
aurel3272fb7da2008-04-27 23:53:45 +00003758 memcpy(buf, p, l);
aurel325b257572008-04-28 08:54:59 +00003759 unlock_user(p, addr, 0);
bellard13eb76e2004-01-24 15:23:36 +00003760 }
3761 len -= l;
3762 buf += l;
3763 addr += l;
3764 }
Paul Brooka68fe892010-03-01 00:08:59 +00003765 return 0;
bellard13eb76e2004-01-24 15:23:36 +00003766}
bellard8df1cd02005-01-28 22:37:22 +00003767
bellard13eb76e2004-01-24 15:23:36 +00003768#else
Anthony Liguoric227f092009-10-01 16:12:16 -05003769void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
bellard13eb76e2004-01-24 15:23:36 +00003770 int len, int is_write)
3771{
3772 int l, io_index;
3773 uint8_t *ptr;
3774 uint32_t val;
Anthony Liguoric227f092009-10-01 16:12:16 -05003775 target_phys_addr_t page;
bellard2e126692004-04-25 21:28:44 +00003776 unsigned long pd;
bellard92e873b2004-05-21 14:52:29 +00003777 PhysPageDesc *p;
ths3b46e622007-09-17 08:09:54 +00003778
bellard13eb76e2004-01-24 15:23:36 +00003779 while (len > 0) {
3780 page = addr & TARGET_PAGE_MASK;
3781 l = (page + TARGET_PAGE_SIZE) - addr;
3782 if (l > len)
3783 l = len;
bellard92e873b2004-05-21 14:52:29 +00003784 p = phys_page_find(page >> TARGET_PAGE_BITS);
bellard13eb76e2004-01-24 15:23:36 +00003785 if (!p) {
3786 pd = IO_MEM_UNASSIGNED;
3787 } else {
3788 pd = p->phys_offset;
3789 }
ths3b46e622007-09-17 08:09:54 +00003790
bellard13eb76e2004-01-24 15:23:36 +00003791 if (is_write) {
bellard3a7d9292005-08-21 09:26:42 +00003792 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
Anthony Liguoric227f092009-10-01 16:12:16 -05003793 target_phys_addr_t addr1 = addr;
bellard13eb76e2004-01-24 15:23:36 +00003794 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00003795 if (p)
aurel326c2934d2009-02-18 21:37:17 +00003796 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard6a00d602005-11-21 23:25:50 +00003797 /* XXX: could force cpu_single_env to NULL to avoid
3798 potential bugs */
aurel326c2934d2009-02-18 21:37:17 +00003799 if (l >= 4 && ((addr1 & 3) == 0)) {
bellard1c213d12005-09-03 10:49:04 +00003800 /* 32 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003801 val = ldl_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003802 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003803 l = 4;
aurel326c2934d2009-02-18 21:37:17 +00003804 } else if (l >= 2 && ((addr1 & 1) == 0)) {
bellard1c213d12005-09-03 10:49:04 +00003805 /* 16 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003806 val = lduw_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003807 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003808 l = 2;
3809 } else {
bellard1c213d12005-09-03 10:49:04 +00003810 /* 8 bit write access */
bellardc27004e2005-01-03 23:35:10 +00003811 val = ldub_p(buf);
aurel326c2934d2009-02-18 21:37:17 +00003812 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
bellard13eb76e2004-01-24 15:23:36 +00003813 l = 1;
3814 }
3815 } else {
bellardb448f2f2004-02-25 23:24:04 +00003816 unsigned long addr1;
3817 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
bellard13eb76e2004-01-24 15:23:36 +00003818 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00003819 ptr = qemu_get_ram_ptr(addr1);
bellard13eb76e2004-01-24 15:23:36 +00003820 memcpy(ptr, buf, l);
bellard3a7d9292005-08-21 09:26:42 +00003821 if (!cpu_physical_memory_is_dirty(addr1)) {
3822 /* invalidate code */
3823 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3824 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09003825 cpu_physical_memory_set_dirty_flags(
3826 addr1, (0xff & ~CODE_DIRTY_FLAG));
bellard3a7d9292005-08-21 09:26:42 +00003827 }
bellard13eb76e2004-01-24 15:23:36 +00003828 }
3829 } else {
ths5fafdf22007-09-16 21:08:06 +00003830 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
bellard2a4188a2006-06-25 21:54:59 +00003831 !(pd & IO_MEM_ROMD)) {
Anthony Liguoric227f092009-10-01 16:12:16 -05003832 target_phys_addr_t addr1 = addr;
bellard13eb76e2004-01-24 15:23:36 +00003833 /* I/O case */
3834 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00003835 if (p)
aurel326c2934d2009-02-18 21:37:17 +00003836 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3837 if (l >= 4 && ((addr1 & 3) == 0)) {
bellard13eb76e2004-01-24 15:23:36 +00003838 /* 32 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003839 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003840 stl_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003841 l = 4;
aurel326c2934d2009-02-18 21:37:17 +00003842 } else if (l >= 2 && ((addr1 & 1) == 0)) {
bellard13eb76e2004-01-24 15:23:36 +00003843 /* 16 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003844 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003845 stw_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003846 l = 2;
3847 } else {
bellard1c213d12005-09-03 10:49:04 +00003848 /* 8 bit read access */
aurel326c2934d2009-02-18 21:37:17 +00003849 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
bellardc27004e2005-01-03 23:35:10 +00003850 stb_p(buf, val);
bellard13eb76e2004-01-24 15:23:36 +00003851 l = 1;
3852 }
3853 } else {
3854 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00003855 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
bellard13eb76e2004-01-24 15:23:36 +00003856 (addr & ~TARGET_PAGE_MASK);
3857 memcpy(buf, ptr, l);
3858 }
3859 }
3860 len -= l;
3861 buf += l;
3862 addr += l;
3863 }
3864}
bellard8df1cd02005-01-28 22:37:22 +00003865
bellardd0ecd2a2006-04-23 17:14:48 +00003866/* used for ROM loading : can write in RAM and ROM */
Anthony Liguoric227f092009-10-01 16:12:16 -05003867void cpu_physical_memory_write_rom(target_phys_addr_t addr,
bellardd0ecd2a2006-04-23 17:14:48 +00003868 const uint8_t *buf, int len)
3869{
3870 int l;
3871 uint8_t *ptr;
Anthony Liguoric227f092009-10-01 16:12:16 -05003872 target_phys_addr_t page;
bellardd0ecd2a2006-04-23 17:14:48 +00003873 unsigned long pd;
3874 PhysPageDesc *p;
ths3b46e622007-09-17 08:09:54 +00003875
bellardd0ecd2a2006-04-23 17:14:48 +00003876 while (len > 0) {
3877 page = addr & TARGET_PAGE_MASK;
3878 l = (page + TARGET_PAGE_SIZE) - addr;
3879 if (l > len)
3880 l = len;
3881 p = phys_page_find(page >> TARGET_PAGE_BITS);
3882 if (!p) {
3883 pd = IO_MEM_UNASSIGNED;
3884 } else {
3885 pd = p->phys_offset;
3886 }
ths3b46e622007-09-17 08:09:54 +00003887
bellardd0ecd2a2006-04-23 17:14:48 +00003888 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
bellard2a4188a2006-06-25 21:54:59 +00003889 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3890 !(pd & IO_MEM_ROMD)) {
bellardd0ecd2a2006-04-23 17:14:48 +00003891 /* do nothing */
3892 } else {
3893 unsigned long addr1;
3894 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3895 /* ROM/RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00003896 ptr = qemu_get_ram_ptr(addr1);
bellardd0ecd2a2006-04-23 17:14:48 +00003897 memcpy(ptr, buf, l);
3898 }
3899 len -= l;
3900 buf += l;
3901 addr += l;
3902 }
3903}
3904
aliguori6d16c2f2009-01-22 16:59:11 +00003905typedef struct {
3906 void *buffer;
Anthony Liguoric227f092009-10-01 16:12:16 -05003907 target_phys_addr_t addr;
3908 target_phys_addr_t len;
aliguori6d16c2f2009-01-22 16:59:11 +00003909} BounceBuffer;
3910
3911static BounceBuffer bounce;
3912
aliguoriba223c22009-01-22 16:59:16 +00003913typedef struct MapClient {
3914 void *opaque;
3915 void (*callback)(void *opaque);
Blue Swirl72cf2d42009-09-12 07:36:22 +00003916 QLIST_ENTRY(MapClient) link;
aliguoriba223c22009-01-22 16:59:16 +00003917} MapClient;
3918
Blue Swirl72cf2d42009-09-12 07:36:22 +00003919static QLIST_HEAD(map_client_list, MapClient) map_client_list
3920 = QLIST_HEAD_INITIALIZER(map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00003921
3922void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3923{
3924 MapClient *client = qemu_malloc(sizeof(*client));
3925
3926 client->opaque = opaque;
3927 client->callback = callback;
Blue Swirl72cf2d42009-09-12 07:36:22 +00003928 QLIST_INSERT_HEAD(&map_client_list, client, link);
aliguoriba223c22009-01-22 16:59:16 +00003929 return client;
3930}
3931
3932void cpu_unregister_map_client(void *_client)
3933{
3934 MapClient *client = (MapClient *)_client;
3935
Blue Swirl72cf2d42009-09-12 07:36:22 +00003936 QLIST_REMOVE(client, link);
Isaku Yamahata34d5e942009-06-26 18:57:18 +09003937 qemu_free(client);
aliguoriba223c22009-01-22 16:59:16 +00003938}
3939
3940static void cpu_notify_map_clients(void)
3941{
3942 MapClient *client;
3943
Blue Swirl72cf2d42009-09-12 07:36:22 +00003944 while (!QLIST_EMPTY(&map_client_list)) {
3945 client = QLIST_FIRST(&map_client_list);
aliguoriba223c22009-01-22 16:59:16 +00003946 client->callback(client->opaque);
Isaku Yamahata34d5e942009-06-26 18:57:18 +09003947 cpu_unregister_map_client(client);
aliguoriba223c22009-01-22 16:59:16 +00003948 }
3949}
3950
aliguori6d16c2f2009-01-22 16:59:11 +00003951/* Map a physical memory region into a host virtual address.
3952 * May map a subset of the requested range, given by and returned in *plen.
3953 * May return NULL if resources needed to perform the mapping are exhausted.
3954 * Use only for reads OR writes - not for read-modify-write operations.
aliguoriba223c22009-01-22 16:59:16 +00003955 * Use cpu_register_map_client() to know when retrying the map operation is
3956 * likely to succeed.
aliguori6d16c2f2009-01-22 16:59:11 +00003957 */
Anthony Liguoric227f092009-10-01 16:12:16 -05003958void *cpu_physical_memory_map(target_phys_addr_t addr,
3959 target_phys_addr_t *plen,
aliguori6d16c2f2009-01-22 16:59:11 +00003960 int is_write)
3961{
Anthony Liguoric227f092009-10-01 16:12:16 -05003962 target_phys_addr_t len = *plen;
3963 target_phys_addr_t done = 0;
aliguori6d16c2f2009-01-22 16:59:11 +00003964 int l;
3965 uint8_t *ret = NULL;
3966 uint8_t *ptr;
Anthony Liguoric227f092009-10-01 16:12:16 -05003967 target_phys_addr_t page;
aliguori6d16c2f2009-01-22 16:59:11 +00003968 unsigned long pd;
3969 PhysPageDesc *p;
3970 unsigned long addr1;
3971
3972 while (len > 0) {
3973 page = addr & TARGET_PAGE_MASK;
3974 l = (page + TARGET_PAGE_SIZE) - addr;
3975 if (l > len)
3976 l = len;
3977 p = phys_page_find(page >> TARGET_PAGE_BITS);
3978 if (!p) {
3979 pd = IO_MEM_UNASSIGNED;
3980 } else {
3981 pd = p->phys_offset;
3982 }
3983
3984 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3985 if (done || bounce.buffer) {
3986 break;
3987 }
3988 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3989 bounce.addr = addr;
3990 bounce.len = l;
3991 if (!is_write) {
Stefan Weil54f7b4a2011-04-10 18:23:39 +02003992 cpu_physical_memory_read(addr, bounce.buffer, l);
aliguori6d16c2f2009-01-22 16:59:11 +00003993 }
3994 ptr = bounce.buffer;
3995 } else {
3996 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
pbrook5579c7f2009-04-11 14:47:08 +00003997 ptr = qemu_get_ram_ptr(addr1);
aliguori6d16c2f2009-01-22 16:59:11 +00003998 }
3999 if (!done) {
4000 ret = ptr;
4001 } else if (ret + done != ptr) {
4002 break;
4003 }
4004
4005 len -= l;
4006 addr += l;
4007 done += l;
4008 }
4009 *plen = done;
4010 return ret;
4011}
4012
4013/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
4014 * Will also mark the memory as dirty if is_write == 1. access_len gives
4015 * the amount of memory that was actually read or written by the caller.
4016 */
Anthony Liguoric227f092009-10-01 16:12:16 -05004017void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
4018 int is_write, target_phys_addr_t access_len)
aliguori6d16c2f2009-01-22 16:59:11 +00004019{
4020 if (buffer != bounce.buffer) {
4021 if (is_write) {
Marcelo Tosattie8902612010-10-11 15:31:19 -03004022 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00004023 while (access_len) {
4024 unsigned l;
4025 l = TARGET_PAGE_SIZE;
4026 if (l > access_len)
4027 l = access_len;
4028 if (!cpu_physical_memory_is_dirty(addr1)) {
4029 /* invalidate code */
4030 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
4031 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004032 cpu_physical_memory_set_dirty_flags(
4033 addr1, (0xff & ~CODE_DIRTY_FLAG));
aliguori6d16c2f2009-01-22 16:59:11 +00004034 }
4035 addr1 += l;
4036 access_len -= l;
4037 }
4038 }
4039 return;
4040 }
4041 if (is_write) {
4042 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
4043 }
Herve Poussineauf8a83242010-01-24 21:23:56 +00004044 qemu_vfree(bounce.buffer);
aliguori6d16c2f2009-01-22 16:59:11 +00004045 bounce.buffer = NULL;
aliguoriba223c22009-01-22 16:59:16 +00004046 cpu_notify_map_clients();
aliguori6d16c2f2009-01-22 16:59:11 +00004047}
bellardd0ecd2a2006-04-23 17:14:48 +00004048
bellard8df1cd02005-01-28 22:37:22 +00004049/* warning: addr must be aligned */
Anthony Liguoric227f092009-10-01 16:12:16 -05004050uint32_t ldl_phys(target_phys_addr_t addr)
bellard8df1cd02005-01-28 22:37:22 +00004051{
4052 int io_index;
4053 uint8_t *ptr;
4054 uint32_t val;
4055 unsigned long pd;
4056 PhysPageDesc *p;
4057
4058 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4059 if (!p) {
4060 pd = IO_MEM_UNASSIGNED;
4061 } else {
4062 pd = p->phys_offset;
4063 }
ths3b46e622007-09-17 08:09:54 +00004064
ths5fafdf22007-09-16 21:08:06 +00004065 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
bellard2a4188a2006-06-25 21:54:59 +00004066 !(pd & IO_MEM_ROMD)) {
bellard8df1cd02005-01-28 22:37:22 +00004067 /* I/O case */
4068 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004069 if (p)
4070 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard8df1cd02005-01-28 22:37:22 +00004071 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4072 } else {
4073 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004074 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
bellard8df1cd02005-01-28 22:37:22 +00004075 (addr & ~TARGET_PAGE_MASK);
4076 val = ldl_p(ptr);
4077 }
4078 return val;
4079}
4080
bellard84b7b8e2005-11-28 21:19:04 +00004081/* warning: addr must be aligned */
Anthony Liguoric227f092009-10-01 16:12:16 -05004082uint64_t ldq_phys(target_phys_addr_t addr)
bellard84b7b8e2005-11-28 21:19:04 +00004083{
4084 int io_index;
4085 uint8_t *ptr;
4086 uint64_t val;
4087 unsigned long pd;
4088 PhysPageDesc *p;
4089
4090 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4091 if (!p) {
4092 pd = IO_MEM_UNASSIGNED;
4093 } else {
4094 pd = p->phys_offset;
4095 }
ths3b46e622007-09-17 08:09:54 +00004096
bellard2a4188a2006-06-25 21:54:59 +00004097 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4098 !(pd & IO_MEM_ROMD)) {
bellard84b7b8e2005-11-28 21:19:04 +00004099 /* I/O case */
4100 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004101 if (p)
4102 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard84b7b8e2005-11-28 21:19:04 +00004103#ifdef TARGET_WORDS_BIGENDIAN
4104 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4105 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4106#else
4107 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4108 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4109#endif
4110 } else {
4111 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004112 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
bellard84b7b8e2005-11-28 21:19:04 +00004113 (addr & ~TARGET_PAGE_MASK);
4114 val = ldq_p(ptr);
4115 }
4116 return val;
4117}
4118
bellardaab33092005-10-30 20:48:42 +00004119/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004120uint32_t ldub_phys(target_phys_addr_t addr)
bellardaab33092005-10-30 20:48:42 +00004121{
4122 uint8_t val;
4123 cpu_physical_memory_read(addr, &val, 1);
4124 return val;
4125}
4126
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004127/* warning: addr must be aligned */
Anthony Liguoric227f092009-10-01 16:12:16 -05004128uint32_t lduw_phys(target_phys_addr_t addr)
bellardaab33092005-10-30 20:48:42 +00004129{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004130 int io_index;
4131 uint8_t *ptr;
4132 uint64_t val;
4133 unsigned long pd;
4134 PhysPageDesc *p;
4135
4136 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4137 if (!p) {
4138 pd = IO_MEM_UNASSIGNED;
4139 } else {
4140 pd = p->phys_offset;
4141 }
4142
4143 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4144 !(pd & IO_MEM_ROMD)) {
4145 /* I/O case */
4146 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4147 if (p)
4148 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4149 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
4150 } else {
4151 /* RAM case */
4152 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4153 (addr & ~TARGET_PAGE_MASK);
4154 val = lduw_p(ptr);
4155 }
4156 return val;
bellardaab33092005-10-30 20:48:42 +00004157}
4158
bellard8df1cd02005-01-28 22:37:22 +00004159/* warning: addr must be aligned. The ram page is not masked as dirty
4160 and the code inside is not invalidated. It is useful if the dirty
4161 bits are used to track modified PTEs */
Anthony Liguoric227f092009-10-01 16:12:16 -05004162void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
bellard8df1cd02005-01-28 22:37:22 +00004163{
4164 int io_index;
4165 uint8_t *ptr;
4166 unsigned long pd;
4167 PhysPageDesc *p;
4168
4169 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4170 if (!p) {
4171 pd = IO_MEM_UNASSIGNED;
4172 } else {
4173 pd = p->phys_offset;
4174 }
ths3b46e622007-09-17 08:09:54 +00004175
bellard3a7d9292005-08-21 09:26:42 +00004176 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
bellard8df1cd02005-01-28 22:37:22 +00004177 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004178 if (p)
4179 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard8df1cd02005-01-28 22:37:22 +00004180 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4181 } else {
aliguori74576192008-10-06 14:02:03 +00004182 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
pbrook5579c7f2009-04-11 14:47:08 +00004183 ptr = qemu_get_ram_ptr(addr1);
bellard8df1cd02005-01-28 22:37:22 +00004184 stl_p(ptr, val);
aliguori74576192008-10-06 14:02:03 +00004185
4186 if (unlikely(in_migration)) {
4187 if (!cpu_physical_memory_is_dirty(addr1)) {
4188 /* invalidate code */
4189 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4190 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004191 cpu_physical_memory_set_dirty_flags(
4192 addr1, (0xff & ~CODE_DIRTY_FLAG));
aliguori74576192008-10-06 14:02:03 +00004193 }
4194 }
bellard8df1cd02005-01-28 22:37:22 +00004195 }
4196}
4197
Anthony Liguoric227f092009-10-01 16:12:16 -05004198void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
j_mayerbc98a7e2007-04-04 07:55:12 +00004199{
4200 int io_index;
4201 uint8_t *ptr;
4202 unsigned long pd;
4203 PhysPageDesc *p;
4204
4205 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4206 if (!p) {
4207 pd = IO_MEM_UNASSIGNED;
4208 } else {
4209 pd = p->phys_offset;
4210 }
ths3b46e622007-09-17 08:09:54 +00004211
j_mayerbc98a7e2007-04-04 07:55:12 +00004212 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4213 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004214 if (p)
4215 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
j_mayerbc98a7e2007-04-04 07:55:12 +00004216#ifdef TARGET_WORDS_BIGENDIAN
4217 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4218 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4219#else
4220 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4221 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4222#endif
4223 } else {
pbrook5579c7f2009-04-11 14:47:08 +00004224 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
j_mayerbc98a7e2007-04-04 07:55:12 +00004225 (addr & ~TARGET_PAGE_MASK);
4226 stq_p(ptr, val);
4227 }
4228}
4229
bellard8df1cd02005-01-28 22:37:22 +00004230/* warning: addr must be aligned */
Anthony Liguoric227f092009-10-01 16:12:16 -05004231void stl_phys(target_phys_addr_t addr, uint32_t val)
bellard8df1cd02005-01-28 22:37:22 +00004232{
4233 int io_index;
4234 uint8_t *ptr;
4235 unsigned long pd;
4236 PhysPageDesc *p;
4237
4238 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4239 if (!p) {
4240 pd = IO_MEM_UNASSIGNED;
4241 } else {
4242 pd = p->phys_offset;
4243 }
ths3b46e622007-09-17 08:09:54 +00004244
bellard3a7d9292005-08-21 09:26:42 +00004245 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
bellard8df1cd02005-01-28 22:37:22 +00004246 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
pbrook8da3ff12008-12-01 18:59:50 +00004247 if (p)
4248 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bellard8df1cd02005-01-28 22:37:22 +00004249 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4250 } else {
4251 unsigned long addr1;
4252 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4253 /* RAM case */
pbrook5579c7f2009-04-11 14:47:08 +00004254 ptr = qemu_get_ram_ptr(addr1);
bellard8df1cd02005-01-28 22:37:22 +00004255 stl_p(ptr, val);
bellard3a7d9292005-08-21 09:26:42 +00004256 if (!cpu_physical_memory_is_dirty(addr1)) {
4257 /* invalidate code */
4258 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4259 /* set dirty bit */
Yoshiaki Tamuraf7c11b52010-03-23 16:39:53 +09004260 cpu_physical_memory_set_dirty_flags(addr1,
4261 (0xff & ~CODE_DIRTY_FLAG));
bellard3a7d9292005-08-21 09:26:42 +00004262 }
bellard8df1cd02005-01-28 22:37:22 +00004263 }
4264}
4265
bellardaab33092005-10-30 20:48:42 +00004266/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004267void stb_phys(target_phys_addr_t addr, uint32_t val)
bellardaab33092005-10-30 20:48:42 +00004268{
4269 uint8_t v = val;
4270 cpu_physical_memory_write(addr, &v, 1);
4271}
4272
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004273/* warning: addr must be aligned */
Anthony Liguoric227f092009-10-01 16:12:16 -05004274void stw_phys(target_phys_addr_t addr, uint32_t val)
bellardaab33092005-10-30 20:48:42 +00004275{
Michael S. Tsirkin733f0b02010-04-06 14:18:19 +03004276 int io_index;
4277 uint8_t *ptr;
4278 unsigned long pd;
4279 PhysPageDesc *p;
4280
4281 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4282 if (!p) {
4283 pd = IO_MEM_UNASSIGNED;
4284 } else {
4285 pd = p->phys_offset;
4286 }
4287
4288 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4289 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4290 if (p)
4291 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4292 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4293 } else {
4294 unsigned long addr1;
4295 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4296 /* RAM case */
4297 ptr = qemu_get_ram_ptr(addr1);
4298 stw_p(ptr, val);
4299 if (!cpu_physical_memory_is_dirty(addr1)) {
4300 /* invalidate code */
4301 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4302 /* set dirty bit */
4303 cpu_physical_memory_set_dirty_flags(addr1,
4304 (0xff & ~CODE_DIRTY_FLAG));
4305 }
4306 }
bellardaab33092005-10-30 20:48:42 +00004307}
4308
4309/* XXX: optimize */
Anthony Liguoric227f092009-10-01 16:12:16 -05004310void stq_phys(target_phys_addr_t addr, uint64_t val)
bellardaab33092005-10-30 20:48:42 +00004311{
4312 val = tswap64(val);
Stefan Weil71d2b722011-03-26 21:06:56 +01004313 cpu_physical_memory_write(addr, &val, 8);
bellardaab33092005-10-30 20:48:42 +00004314}
4315
aliguori5e2972f2009-03-28 17:51:36 +00004316/* virtual memory access for debug (includes writing to ROM) */
ths5fafdf22007-09-16 21:08:06 +00004317int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
bellardb448f2f2004-02-25 23:24:04 +00004318 uint8_t *buf, int len, int is_write)
bellard13eb76e2004-01-24 15:23:36 +00004319{
4320 int l;
Anthony Liguoric227f092009-10-01 16:12:16 -05004321 target_phys_addr_t phys_addr;
j_mayer9b3c35e2007-04-07 11:21:28 +00004322 target_ulong page;
bellard13eb76e2004-01-24 15:23:36 +00004323
4324 while (len > 0) {
4325 page = addr & TARGET_PAGE_MASK;
4326 phys_addr = cpu_get_phys_page_debug(env, page);
4327 /* if no physical page mapped, return an error */
4328 if (phys_addr == -1)
4329 return -1;
4330 l = (page + TARGET_PAGE_SIZE) - addr;
4331 if (l > len)
4332 l = len;
aliguori5e2972f2009-03-28 17:51:36 +00004333 phys_addr += (addr & ~TARGET_PAGE_MASK);
aliguori5e2972f2009-03-28 17:51:36 +00004334 if (is_write)
4335 cpu_physical_memory_write_rom(phys_addr, buf, l);
4336 else
aliguori5e2972f2009-03-28 17:51:36 +00004337 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
bellard13eb76e2004-01-24 15:23:36 +00004338 len -= l;
4339 buf += l;
4340 addr += l;
4341 }
4342 return 0;
4343}
Paul Brooka68fe892010-03-01 00:08:59 +00004344#endif
bellard13eb76e2004-01-24 15:23:36 +00004345
pbrook2e70f6e2008-06-29 01:03:05 +00004346/* in deterministic execution mode, instructions doing device I/Os
4347 must be at the end of the TB */
4348void cpu_io_recompile(CPUState *env, void *retaddr)
4349{
4350 TranslationBlock *tb;
4351 uint32_t n, cflags;
4352 target_ulong pc, cs_base;
4353 uint64_t flags;
4354
4355 tb = tb_find_pc((unsigned long)retaddr);
4356 if (!tb) {
4357 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4358 retaddr);
4359 }
4360 n = env->icount_decr.u16.low + tb->icount;
Stefan Weil618ba8e2011-04-18 06:39:53 +00004361 cpu_restore_state(tb, env, (unsigned long)retaddr);
pbrook2e70f6e2008-06-29 01:03:05 +00004362 /* Calculate how many instructions had been executed before the fault
thsbf20dc02008-06-30 17:22:19 +00004363 occurred. */
pbrook2e70f6e2008-06-29 01:03:05 +00004364 n = n - env->icount_decr.u16.low;
4365 /* Generate a new TB ending on the I/O insn. */
4366 n++;
4367 /* On MIPS and SH, delay slot instructions can only be restarted if
4368 they were already the first instruction in the TB. If this is not
thsbf20dc02008-06-30 17:22:19 +00004369 the first instruction in a TB then re-execute the preceding
pbrook2e70f6e2008-06-29 01:03:05 +00004370 branch. */
4371#if defined(TARGET_MIPS)
4372 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4373 env->active_tc.PC -= 4;
4374 env->icount_decr.u16.low++;
4375 env->hflags &= ~MIPS_HFLAG_BMASK;
4376 }
4377#elif defined(TARGET_SH4)
4378 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4379 && n > 1) {
4380 env->pc -= 2;
4381 env->icount_decr.u16.low++;
4382 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4383 }
4384#endif
4385 /* This should never happen. */
4386 if (n > CF_COUNT_MASK)
4387 cpu_abort(env, "TB too big during recompile");
4388
4389 cflags = n | CF_LAST_IO;
4390 pc = tb->pc;
4391 cs_base = tb->cs_base;
4392 flags = tb->flags;
4393 tb_phys_invalidate(tb, -1);
4394 /* FIXME: In theory this could raise an exception. In practice
4395 we have already translated the block once so it's probably ok. */
4396 tb_gen_code(env, pc, cs_base, flags, cflags);
thsbf20dc02008-06-30 17:22:19 +00004397 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
pbrook2e70f6e2008-06-29 01:03:05 +00004398 the first in the TB) then we end up generating a whole new TB and
4399 repeating the fault, which is horribly inefficient.
4400 Better would be to execute just this insn uncached, or generate a
4401 second new TB. */
4402 cpu_resume_from_signal(env, NULL);
4403}
4404
Paul Brookb3755a92010-03-12 16:54:58 +00004405#if !defined(CONFIG_USER_ONLY)
4406
Stefan Weil055403b2010-10-22 23:03:32 +02004407void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
bellarde3db7222005-01-26 22:00:47 +00004408{
4409 int i, target_code_size, max_target_code_size;
4410 int direct_jmp_count, direct_jmp2_count, cross_page;
4411 TranslationBlock *tb;
ths3b46e622007-09-17 08:09:54 +00004412
bellarde3db7222005-01-26 22:00:47 +00004413 target_code_size = 0;
4414 max_target_code_size = 0;
4415 cross_page = 0;
4416 direct_jmp_count = 0;
4417 direct_jmp2_count = 0;
4418 for(i = 0; i < nb_tbs; i++) {
4419 tb = &tbs[i];
4420 target_code_size += tb->size;
4421 if (tb->size > max_target_code_size)
4422 max_target_code_size = tb->size;
4423 if (tb->page_addr[1] != -1)
4424 cross_page++;
4425 if (tb->tb_next_offset[0] != 0xffff) {
4426 direct_jmp_count++;
4427 if (tb->tb_next_offset[1] != 0xffff) {
4428 direct_jmp2_count++;
4429 }
4430 }
4431 }
4432 /* XXX: avoid using doubles ? */
bellard57fec1f2008-02-01 10:50:11 +00004433 cpu_fprintf(f, "Translation buffer state:\n");
Stefan Weil055403b2010-10-22 23:03:32 +02004434 cpu_fprintf(f, "gen code size %td/%ld\n",
bellard26a5f132008-05-28 12:30:31 +00004435 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4436 cpu_fprintf(f, "TB count %d/%d\n",
4437 nb_tbs, code_gen_max_blocks);
ths5fafdf22007-09-16 21:08:06 +00004438 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
bellarde3db7222005-01-26 22:00:47 +00004439 nb_tbs ? target_code_size / nb_tbs : 0,
4440 max_target_code_size);
Stefan Weil055403b2010-10-22 23:03:32 +02004441 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
bellarde3db7222005-01-26 22:00:47 +00004442 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4443 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
ths5fafdf22007-09-16 21:08:06 +00004444 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4445 cross_page,
bellarde3db7222005-01-26 22:00:47 +00004446 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4447 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
ths5fafdf22007-09-16 21:08:06 +00004448 direct_jmp_count,
bellarde3db7222005-01-26 22:00:47 +00004449 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4450 direct_jmp2_count,
4451 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
bellard57fec1f2008-02-01 10:50:11 +00004452 cpu_fprintf(f, "\nStatistics:\n");
bellarde3db7222005-01-26 22:00:47 +00004453 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4454 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4455 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
bellardb67d9a52008-05-23 09:57:34 +00004456 tcg_dump_info(f, cpu_fprintf);
bellarde3db7222005-01-26 22:00:47 +00004457}
4458
bellard61382a52003-10-27 21:22:23 +00004459#define MMUSUFFIX _cmmu
4460#define GETPC() NULL
4461#define env cpu_single_env
bellardb769d8f2004-10-03 15:07:13 +00004462#define SOFTMMU_CODE_ACCESS
bellard61382a52003-10-27 21:22:23 +00004463
4464#define SHIFT 0
4465#include "softmmu_template.h"
4466
4467#define SHIFT 1
4468#include "softmmu_template.h"
4469
4470#define SHIFT 2
4471#include "softmmu_template.h"
4472
4473#define SHIFT 3
4474#include "softmmu_template.h"
4475
4476#undef env
4477
4478#endif