diff --git a/.gitignore b/.gitignore
index 8a52709..c658613 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@
 /config-host.*
 /config-target.*
 /config.status
+/config-temp
 /trace/generated-tracers.h
 /trace/generated-tracers.c
 /trace/generated-tracers-dtrace.h
diff --git a/MAINTAINERS b/MAINTAINERS
index 1de05f0..51a6f51 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -243,8 +243,8 @@
 F: hw/*/exynos*
 
 Calxeda Highbank
-M: Mark Langsdorf <mark.langsdorf@calxeda.com>
-S: Supported
+M: Rob Herring <robh@kernel.org>
+S: Maintained
 F: hw/arm/highbank.c
 F: hw/net/xgmac.c
 
diff --git a/arch_init.c b/arch_init.c
index 685ba0e..9f1a174 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -975,12 +975,12 @@
     xh_len = qemu_get_be16(f);
 
     if (xh_flags != ENCODING_FLAG_XBZRLE) {
-        fprintf(stderr, "Failed to load XBZRLE page - wrong compression!\n");
+        error_report("Failed to load XBZRLE page - wrong compression!");
         return -1;
     }
 
     if (xh_len > TARGET_PAGE_SIZE) {
-        fprintf(stderr, "Failed to load XBZRLE page - len overflow!\n");
+        error_report("Failed to load XBZRLE page - len overflow!");
         return -1;
     }
     /* load data and decode */
@@ -989,7 +989,7 @@
     /* decode RLE */
     if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host,
                              TARGET_PAGE_SIZE) == -1) {
-        fprintf(stderr, "Failed to load XBZRLE page - decode error!\n");
+        error_report("Failed to load XBZRLE page - decode error!");
         return -1;
     }
 
@@ -1006,7 +1006,7 @@
 
     if (flags & RAM_SAVE_FLAG_CONTINUE) {
         if (!block) {
-            fprintf(stderr, "Ack, bad migration stream!\n");
+            error_report("Ack, bad migration stream!");
             return NULL;
         }
 
@@ -1022,7 +1022,7 @@
             return memory_region_get_ram_ptr(block->mr) + offset;
     }
 
-    fprintf(stderr, "Can't find block %s!\n", id);
+    error_report("Can't find block %s!", id);
     return NULL;
 }
 
@@ -1075,10 +1075,9 @@
                 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
                     if (!strncmp(id, block->idstr, sizeof(id))) {
                         if (block->length != length) {
-                            fprintf(stderr,
-                                    "Length mismatch: %s: " RAM_ADDR_FMT
-                                    " in != " RAM_ADDR_FMT "\n", id, length,
-                                    block->length);
+                            error_report("Length mismatch: %s: " RAM_ADDR_FMT
+                                         " in != " RAM_ADDR_FMT, id, length,
+                                         block->length);
                             ret =  -EINVAL;
                             goto done;
                         }
@@ -1087,8 +1086,8 @@
                 }
 
                 if (!block) {
-                    fprintf(stderr, "Unknown ramblock \"%s\", cannot "
-                            "accept migration\n", id);
+                    error_report("Unknown ramblock \"%s\", cannot "
+                                 "accept migration", id);
                     ret = -EINVAL;
                     goto done;
                 }
@@ -1243,12 +1242,11 @@
 
             if (!c->name) {
                 if (l > 80) {
-                    fprintf(stderr,
-                            "Unknown sound card name (too big to show)\n");
+                    error_report("Unknown sound card name (too big to show)");
                 }
                 else {
-                    fprintf(stderr, "Unknown sound card name `%.*s'\n",
-                            (int) l, p);
+                    error_report("Unknown sound card name `%.*s'",
+                                 (int) l, p);
                 }
                 bad_card = 1;
             }
@@ -1271,13 +1269,13 @@
         if (c->enabled) {
             if (c->isa) {
                 if (!isa_bus) {
-                    fprintf(stderr, "ISA bus not available for %s\n", c->name);
+                    error_report("ISA bus not available for %s", c->name);
                     exit(1);
                 }
                 c->init.init_isa(isa_bus);
             } else {
                 if (!pci_bus) {
-                    fprintf(stderr, "PCI bus not available for %s\n", c->name);
+                    error_report("PCI bus not available for %s", c->name);
                     exit(1);
                 }
                 c->init.init_pci(pci_bus);
diff --git a/audio/spiceaudio.c b/audio/spiceaudio.c
index fceee50..7b79bed 100644
--- a/audio/spiceaudio.c
+++ b/audio/spiceaudio.c
@@ -105,7 +105,7 @@
     bytes = muldiv64 (ticks, info->bytes_per_second, get_ticks_per_sec ());
     samples = (bytes - rate->bytes_sent) >> info->shift;
     if (samples < 0 || samples > 65536) {
-        fprintf (stderr, "Resetting rate control (%" PRId64 " samples)\n", samples);
+        error_report("Resetting rate control (%" PRId64 " samples)", samples);
         rate_start (rate);
         samples = 0;
     }
diff --git a/audio/wavcapture.c b/audio/wavcapture.c
index 9d94623..6f6d792 100644
--- a/audio/wavcapture.c
+++ b/audio/wavcapture.c
@@ -63,8 +63,7 @@
         }
     doclose:
         if (fclose (wav->f)) {
-            fprintf (stderr, "wav_destroy: fclose failed: %s",
-                     strerror (errno));
+            error_report("wav_destroy: fclose failed: %s", strerror(errno));
         }
     }
 
diff --git a/block-migration.c b/block-migration.c
index 56951e0..1656270 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -59,6 +59,7 @@
     unsigned long *aio_bitmap;
     int64_t completed_sectors;
     BdrvDirtyBitmap *dirty_bitmap;
+    Error *blocker;
 } BlkMigDevState;
 
 typedef struct BlkMigBlock {
@@ -361,7 +362,8 @@
         bmds->completed_sectors = 0;
         bmds->shared_base = block_mig_state.shared_base;
         alloc_aio_bitmap(bmds);
-        bdrv_set_in_use(bs, 1);
+        error_setg(&bmds->blocker, "block device is in use by migration");
+        bdrv_op_block_all(bs, bmds->blocker);
         bdrv_ref(bs);
 
         block_mig_state.total_sector_sum += sectors;
@@ -599,7 +601,8 @@
     blk_mig_lock();
     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
         QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry);
-        bdrv_set_in_use(bmds->bs, 0);
+        bdrv_op_unblock_all(bmds->bs, bmds->blocker);
+        error_free(bmds->blocker);
         bdrv_unref(bmds->bs);
         g_free(bmds->aio_bitmap);
         g_free(bmds);
diff --git a/block.c b/block.c
index c90c71a..310ea89 100644
--- a/block.c
+++ b/block.c
@@ -335,6 +335,7 @@
 BlockDriverState *bdrv_new(const char *device_name, Error **errp)
 {
     BlockDriverState *bs;
+    int i;
 
     if (bdrv_find(device_name)) {
         error_setg(errp, "Device with id '%s' already exists",
@@ -353,6 +354,9 @@
     if (device_name[0] != '\0') {
         QTAILQ_INSERT_TAIL(&bdrv_states, bs, device_list);
     }
+    for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
+        QLIST_INIT(&bs->op_blockers[i]);
+    }
     bdrv_iostatus_disable(bs);
     notifier_list_init(&bs->close_notifiers);
     notifier_with_return_list_init(&bs->before_write_notifiers);
@@ -1090,6 +1094,37 @@
     return ret;
 }
 
+void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd)
+{
+
+    if (bs->backing_hd) {
+        assert(bs->backing_blocker);
+        bdrv_op_unblock_all(bs->backing_hd, bs->backing_blocker);
+    } else if (backing_hd) {
+        error_setg(&bs->backing_blocker,
+                   "device is used as backing hd of '%s'",
+                   bs->device_name);
+    }
+
+    bs->backing_hd = backing_hd;
+    if (!backing_hd) {
+        error_free(bs->backing_blocker);
+        bs->backing_blocker = NULL;
+        goto out;
+    }
+    bs->open_flags &= ~BDRV_O_NO_BACKING;
+    pstrcpy(bs->backing_file, sizeof(bs->backing_file), backing_hd->filename);
+    pstrcpy(bs->backing_format, sizeof(bs->backing_format),
+            backing_hd->drv ? backing_hd->drv->format_name : "");
+
+    bdrv_op_block_all(bs->backing_hd, bs->backing_blocker);
+    /* Otherwise we won't be able to commit due to check in bdrv_commit */
+    bdrv_op_unblock(bs->backing_hd, BLOCK_OP_TYPE_COMMIT,
+                    bs->backing_blocker);
+out:
+    bdrv_refresh_limits(bs);
+}
+
 /*
  * Opens the backing file for a BlockDriverState if not yet open
  *
@@ -1103,6 +1138,7 @@
     char *backing_filename = g_malloc0(PATH_MAX);
     int ret = 0;
     BlockDriver *back_drv = NULL;
+    BlockDriverState *backing_hd;
     Error *local_err = NULL;
 
     if (bs->backing_hd != NULL) {
@@ -1125,30 +1161,26 @@
         bdrv_get_full_backing_filename(bs, backing_filename, PATH_MAX);
     }
 
+    backing_hd = bdrv_new("", errp);
+
     if (bs->backing_format[0] != '\0') {
         back_drv = bdrv_find_format(bs->backing_format);
     }
 
     assert(bs->backing_hd == NULL);
-    ret = bdrv_open(&bs->backing_hd,
+    ret = bdrv_open(&backing_hd,
                     *backing_filename ? backing_filename : NULL, NULL, options,
                     bdrv_backing_flags(bs->open_flags), back_drv, &local_err);
     if (ret < 0) {
-        bs->backing_hd = NULL;
+        bdrv_unref(backing_hd);
+        backing_hd = NULL;
         bs->open_flags |= BDRV_O_NO_BACKING;
         error_setg(errp, "Could not open backing file: %s",
                    error_get_pretty(local_err));
         error_free(local_err);
         goto free_exit;
     }
-
-    if (bs->backing_hd->file) {
-        pstrcpy(bs->backing_file, sizeof(bs->backing_file),
-                bs->backing_hd->file->filename);
-    }
-
-    /* Recalculate the BlockLimits with the backing file */
-    bdrv_refresh_limits(bs);
+    bdrv_set_backing_hd(bs, backing_hd);
 
 free_exit:
     g_free(backing_filename);
@@ -1196,6 +1228,7 @@
                        bdref_key);
             ret = -EINVAL;
         }
+        QDECREF(image_options);
         goto done;
     }
 
@@ -1274,6 +1307,33 @@
     g_free(tmp_filename);
 }
 
+static QDict *parse_json_filename(const char *filename, Error **errp)
+{
+    QObject *options_obj;
+    QDict *options;
+    int ret;
+
+    ret = strstart(filename, "json:", &filename);
+    assert(ret);
+
+    options_obj = qobject_from_json(filename);
+    if (!options_obj) {
+        error_setg(errp, "Could not parse the JSON options");
+        return NULL;
+    }
+
+    if (qobject_type(options_obj) != QTYPE_QDICT) {
+        qobject_decref(options_obj);
+        error_setg(errp, "Invalid JSON object given");
+        return NULL;
+    }
+
+    options = qobject_to_qdict(options_obj);
+    qdict_flatten(options);
+
+    return options;
+}
+
 /*
  * Opens a disk image (raw, qcow2, vmdk, ...)
  *
@@ -1337,6 +1397,20 @@
         options = qdict_new();
     }
 
+    if (filename && g_str_has_prefix(filename, "json:")) {
+        QDict *json_options = parse_json_filename(filename, &local_err);
+        if (local_err) {
+            ret = -EINVAL;
+            goto fail;
+        }
+
+        /* Options given in the filename have lower priority than options
+         * specified directly */
+        qdict_join(options, json_options, false);
+        QDECREF(json_options);
+        filename = NULL;
+    }
+
     bs->options = options;
     options = qdict_clone_shallow(options);
 
@@ -1743,8 +1817,9 @@
 
     if (bs->drv) {
         if (bs->backing_hd) {
-            bdrv_unref(bs->backing_hd);
-            bs->backing_hd = NULL;
+            BlockDriverState *backing_hd = bs->backing_hd;
+            bdrv_set_backing_hd(bs, NULL);
+            bdrv_unref(backing_hd);
         }
         bs->drv->bdrv_close(bs);
         g_free(bs->opaque);
@@ -1904,13 +1979,14 @@
     bs_dest->refcnt             = bs_src->refcnt;
 
     /* job */
-    bs_dest->in_use             = bs_src->in_use;
     bs_dest->job                = bs_src->job;
 
     /* keep the same entry in bdrv_states */
     pstrcpy(bs_dest->device_name, sizeof(bs_dest->device_name),
             bs_src->device_name);
     bs_dest->device_list = bs_src->device_list;
+    memcpy(bs_dest->op_blockers, bs_src->op_blockers,
+           sizeof(bs_dest->op_blockers));
 }
 
 /*
@@ -1945,7 +2021,6 @@
     assert(QLIST_EMPTY(&bs_new->dirty_bitmaps));
     assert(bs_new->job == NULL);
     assert(bs_new->dev == NULL);
-    assert(bs_new->in_use == 0);
     assert(bs_new->io_limits_enabled == false);
     assert(!throttle_have_timer(&bs_new->throttle_state));
 
@@ -1964,7 +2039,6 @@
     /* Check a few fields that should remain attached to the device */
     assert(bs_new->dev == NULL);
     assert(bs_new->job == NULL);
-    assert(bs_new->in_use == 0);
     assert(bs_new->io_limits_enabled == false);
     assert(!throttle_have_timer(&bs_new->throttle_state));
 
@@ -1997,19 +2071,14 @@
 
     /* The contents of 'tmp' will become bs_top, as we are
      * swapping bs_new and bs_top contents. */
-    bs_top->backing_hd = bs_new;
-    bs_top->open_flags &= ~BDRV_O_NO_BACKING;
-    pstrcpy(bs_top->backing_file, sizeof(bs_top->backing_file),
-            bs_new->filename);
-    pstrcpy(bs_top->backing_format, sizeof(bs_top->backing_format),
-            bs_new->drv ? bs_new->drv->format_name : "");
+    bdrv_set_backing_hd(bs_top, bs_new);
 }
 
 static void bdrv_delete(BlockDriverState *bs)
 {
     assert(!bs->dev);
     assert(!bs->job);
-    assert(!bs->in_use);
+    assert(bdrv_op_blocker_is_empty(bs));
     assert(!bs->refcnt);
     assert(QLIST_EMPTY(&bs->dirty_bitmaps));
 
@@ -2191,7 +2260,8 @@
         return -ENOTSUP;
     }
 
-    if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, NULL) ||
+        bdrv_op_is_blocked(bs->backing_hd, BLOCK_OP_TYPE_COMMIT, NULL)) {
         return -EBUSY;
     }
 
@@ -2595,13 +2665,11 @@
     if (ret) {
         goto exit;
     }
-    new_top_bs->backing_hd = base_bs;
-
-    bdrv_refresh_limits(new_top_bs);
+    bdrv_set_backing_hd(new_top_bs, base_bs);
 
     QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
         /* so that bdrv_close() does not recursively close the chain */
-        intermediate_state->bs->backing_hd = NULL;
+        bdrv_set_backing_hd(intermediate_state->bs, NULL);
         bdrv_unref(intermediate_state->bs);
     }
     ret = 0;
@@ -3248,6 +3316,15 @@
 
     ret = notifier_with_return_list_notify(&bs->before_write_notifiers, req);
 
+    if (!ret && bs->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF &&
+        !(flags & BDRV_REQ_ZERO_WRITE) && drv->bdrv_co_write_zeroes &&
+        qemu_iovec_is_zero(qiov)) {
+        flags |= BDRV_REQ_ZERO_WRITE;
+        if (bs->detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP) {
+            flags |= BDRV_REQ_MAY_UNMAP;
+        }
+    }
+
     if (ret < 0) {
         /* Do nothing, write notifier decided to fail this request */
     } else if (flags & BDRV_REQ_ZERO_WRITE) {
@@ -3444,8 +3521,9 @@
         return -ENOTSUP;
     if (bs->read_only)
         return -EACCES;
-    if (bdrv_in_use(bs))
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_RESIZE, NULL)) {
         return -EBUSY;
+    }
     ret = drv->bdrv_truncate(bs, offset);
     if (ret == 0) {
         ret = refresh_total_sectors(bs, offset >> BDRV_SECTOR_BITS);
@@ -3864,7 +3942,7 @@
 
     if (!bs->drv->bdrv_co_get_block_status) {
         *pnum = nb_sectors;
-        ret = BDRV_BLOCK_DATA;
+        ret = BDRV_BLOCK_DATA | BDRV_BLOCK_ALLOCATED;
         if (bs->drv->protocol_name) {
             ret |= BDRV_BLOCK_OFFSET_VALID | (sector_num * BDRV_SECTOR_SIZE);
         }
@@ -3883,6 +3961,10 @@
                                      *pnum, pnum);
     }
 
+    if (ret & (BDRV_BLOCK_DATA | BDRV_BLOCK_ZERO)) {
+        ret |= BDRV_BLOCK_ALLOCATED;
+    }
+
     if (!(ret & BDRV_BLOCK_DATA) && !(ret & BDRV_BLOCK_ZERO)) {
         if (bdrv_unallocated_blocks_are_zero(bs)) {
             ret |= BDRV_BLOCK_ZERO;
@@ -3959,9 +4041,7 @@
     if (ret < 0) {
         return ret;
     }
-    return
-        (ret & BDRV_BLOCK_DATA) ||
-        ((ret & BDRV_BLOCK_ZERO) && !bdrv_has_zero_init(bs));
+    return (ret & BDRV_BLOCK_ALLOCATED);
 }
 
 /*
@@ -5273,15 +5353,74 @@
     }
 }
 
-void bdrv_set_in_use(BlockDriverState *bs, int in_use)
+struct BdrvOpBlocker {
+    Error *reason;
+    QLIST_ENTRY(BdrvOpBlocker) list;
+};
+
+bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp)
 {
-    assert(bs->in_use != in_use);
-    bs->in_use = in_use;
+    BdrvOpBlocker *blocker;
+    assert((int) op >= 0 && op < BLOCK_OP_TYPE_MAX);
+    if (!QLIST_EMPTY(&bs->op_blockers[op])) {
+        blocker = QLIST_FIRST(&bs->op_blockers[op]);
+        if (errp) {
+            error_setg(errp, "Device '%s' is busy: %s",
+                       bs->device_name, error_get_pretty(blocker->reason));
+        }
+        return true;
+    }
+    return false;
 }
 
-int bdrv_in_use(BlockDriverState *bs)
+void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason)
 {
-    return bs->in_use;
+    BdrvOpBlocker *blocker;
+    assert((int) op >= 0 && op < BLOCK_OP_TYPE_MAX);
+
+    blocker = g_malloc0(sizeof(BdrvOpBlocker));
+    blocker->reason = reason;
+    QLIST_INSERT_HEAD(&bs->op_blockers[op], blocker, list);
+}
+
+void bdrv_op_unblock(BlockDriverState *bs, BlockOpType op, Error *reason)
+{
+    BdrvOpBlocker *blocker, *next;
+    assert((int) op >= 0 && op < BLOCK_OP_TYPE_MAX);
+    QLIST_FOREACH_SAFE(blocker, &bs->op_blockers[op], list, next) {
+        if (blocker->reason == reason) {
+            QLIST_REMOVE(blocker, list);
+            g_free(blocker);
+        }
+    }
+}
+
+void bdrv_op_block_all(BlockDriverState *bs, Error *reason)
+{
+    int i;
+    for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
+        bdrv_op_block(bs, i, reason);
+    }
+}
+
+void bdrv_op_unblock_all(BlockDriverState *bs, Error *reason)
+{
+    int i;
+    for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
+        bdrv_op_unblock(bs, i, reason);
+    }
+}
+
+bool bdrv_op_blocker_is_empty(BlockDriverState *bs)
+{
+    int i;
+
+    for (i = 0; i < BLOCK_OP_TYPE_MAX; i++) {
+        if (!QLIST_EMPTY(&bs->op_blockers[i])) {
+            return false;
+        }
+    }
+    return true;
 }
 
 void bdrv_iostatus_enable(BlockDriverState *bs)
diff --git a/block/curl.c b/block/curl.c
index d2f1084..f491b0b 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -23,6 +23,7 @@
  */
 #include "qemu-common.h"
 #include "block/block_int.h"
+#include "qapi/qmp/qbool.h"
 #include <curl/curl.h>
 
 // #define DEBUG
@@ -37,6 +38,21 @@
 #if LIBCURL_VERSION_NUM >= 0x071000
 /* The multi interface timer callback was introduced in 7.16.0 */
 #define NEED_CURL_TIMER_CALLBACK
+#define HAVE_SOCKET_ACTION
+#endif
+
+#ifndef HAVE_SOCKET_ACTION
+/* If curl_multi_socket_action isn't available, define it statically here in
+ * terms of curl_multi_socket. Note that ev_bitmask will be ignored, which is
+ * less efficient but still safe. */
+static CURLMcode __curl_multi_socket_action(CURLM *multi_handle,
+                                            curl_socket_t sockfd,
+                                            int ev_bitmask,
+                                            int *running_handles)
+{
+    return curl_multi_socket(multi_handle, sockfd, running_handles);
+}
+#define curl_multi_socket_action __curl_multi_socket_action
 #endif
 
 #define PROTOCOLS (CURLPROTO_HTTP | CURLPROTO_HTTPS | \
@@ -46,12 +62,16 @@
 #define CURL_NUM_STATES 8
 #define CURL_NUM_ACB    8
 #define SECTOR_SIZE     512
-#define READ_AHEAD_SIZE (256 * 1024)
+#define READ_AHEAD_DEFAULT (256 * 1024)
 
 #define FIND_RET_NONE   0
 #define FIND_RET_OK     1
 #define FIND_RET_WAIT   2
 
+#define CURL_BLOCK_OPT_URL       "url"
+#define CURL_BLOCK_OPT_READAHEAD "readahead"
+#define CURL_BLOCK_OPT_SSLVERIFY "sslverify"
+
 struct BDRVCURLState;
 
 typedef struct CURLAIOCB {
@@ -88,6 +108,7 @@
     CURLState states[CURL_NUM_STATES];
     char *url;
     size_t readahead_size;
+    bool sslverify;
     bool accept_range;
 } BDRVCURLState;
 
@@ -354,6 +375,8 @@
             return NULL;
         }
         curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
+        curl_easy_setopt(state->curl, CURLOPT_SSL_VERIFYPEER,
+                         (long) s->sslverify);
         curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
         curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION,
                          (void *)curl_read_cb);
@@ -396,43 +419,7 @@
 static void curl_parse_filename(const char *filename, QDict *options,
                                 Error **errp)
 {
-
-    #define RA_OPTSTR ":readahead="
-    char *file;
-    char *ra;
-    const char *ra_val;
-    int parse_state = 0;
-
-    file = g_strdup(filename);
-
-    /* Parse a trailing ":readahead=#:" param, if present. */
-    ra = file + strlen(file) - 1;
-    while (ra >= file) {
-        if (parse_state == 0) {
-            if (*ra == ':') {
-                parse_state++;
-            } else {
-                break;
-            }
-        } else if (parse_state == 1) {
-            if (*ra > '9' || *ra < '0') {
-                char *opt_start = ra - strlen(RA_OPTSTR) + 1;
-                if (opt_start > file &&
-                    strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
-                    ra_val = ra + 1;
-                    ra -= strlen(RA_OPTSTR) - 1;
-                    *ra = '\0';
-                    qdict_put(options, "readahead", qstring_from_str(ra_val));
-                }
-                break;
-            }
-        }
-        ra--;
-    }
-
-    qdict_put(options, "url", qstring_from_str(file));
-
-    g_free(file);
+    qdict_put(options, CURL_BLOCK_OPT_URL, qstring_from_str(filename));
 }
 
 static QemuOptsList runtime_opts = {
@@ -440,15 +427,20 @@
     .head = QTAILQ_HEAD_INITIALIZER(runtime_opts.head),
     .desc = {
         {
-            .name = "url",
+            .name = CURL_BLOCK_OPT_URL,
             .type = QEMU_OPT_STRING,
             .help = "URL to open",
         },
         {
-            .name = "readahead",
+            .name = CURL_BLOCK_OPT_READAHEAD,
             .type = QEMU_OPT_SIZE,
             .help = "Readahead size",
         },
+        {
+            .name = CURL_BLOCK_OPT_SSLVERIFY,
+            .type = QEMU_OPT_BOOL,
+            .help = "Verify SSL certificate"
+        },
         { /* end of list */ }
     },
 };
@@ -477,14 +469,17 @@
         goto out_noclean;
     }
 
-    s->readahead_size = qemu_opt_get_size(opts, "readahead", READ_AHEAD_SIZE);
+    s->readahead_size = qemu_opt_get_size(opts, CURL_BLOCK_OPT_READAHEAD,
+                                          READ_AHEAD_DEFAULT);
     if ((s->readahead_size & 0x1ff) != 0) {
         error_setg(errp, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512",
                    s->readahead_size);
         goto out_noclean;
     }
 
-    file = qemu_opt_get(opts, "url");
+    s->sslverify = qemu_opt_get_bool(opts, CURL_BLOCK_OPT_SSLVERIFY, true);
+
+    file = qemu_opt_get(opts, CURL_BLOCK_OPT_URL);
     if (file == NULL) {
         error_setg(errp, "curl block driver requires an 'url' option");
         goto out_noclean;
diff --git a/block/iscsi.c b/block/iscsi.c
index d649424..3892cc5 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -381,6 +381,7 @@
 }
 
 
+#if defined(LIBISCSI_FEATURE_IOVECTOR)
 static bool iscsi_allocationmap_is_allocated(IscsiLun *iscsilun,
                                              int64_t sector_num, int nb_sectors)
 {
@@ -393,9 +394,6 @@
                            sector_num / iscsilun->cluster_sectors) == size);
 }
 
-
-#if defined(LIBISCSI_FEATURE_IOVECTOR)
-
 static int64_t coroutine_fn iscsi_co_get_block_status(BlockDriverState *bs,
                                                   int64_t sector_num,
                                                   int nb_sectors, int *pnum)
diff --git a/block/mirror.c b/block/mirror.c
index 1c38aa8..94c8661 100644
--- a/block/mirror.c
+++ b/block/mirror.c
@@ -498,7 +498,7 @@
             /* drop the bs loop chain formed by the swap: break the loop then
              * trigger the unref from the top one */
             BlockDriverState *p = s->base->backing_hd;
-            s->base->backing_hd = NULL;
+            bdrv_set_backing_hd(s->base, NULL);
             bdrv_unref(p);
         }
     }
diff --git a/block/qapi.c b/block/qapi.c
index af11445..97e1641 100644
--- a/block/qapi.c
+++ b/block/qapi.c
@@ -50,6 +50,7 @@
     }
 
     info->backing_file_depth = bdrv_get_backing_file_depth(bs);
+    info->detect_zeroes = bs->detect_zeroes;
 
     if (bs->io_limits_enabled) {
         ThrottleConfig cfg;
@@ -474,6 +475,7 @@
         case QTYPE_QERROR: {
             QString *value = qerror_human((QError *)obj);
             func_fprintf(f, "%s", qstring_get_str(value));
+            QDECREF(value);
             break;
         }
         case QTYPE_NONE:
diff --git a/block/qcow.c b/block/qcow.c
index 937dd6d..7fd57d7 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -48,9 +48,10 @@
     uint64_t size; /* in bytes */
     uint8_t cluster_bits;
     uint8_t l2_bits;
+    uint16_t padding;
     uint32_t crypt_method;
     uint64_t l1_table_offset;
-} QCowHeader;
+} QEMU_PACKED QCowHeader;
 
 #define L2_CACHE_SIZE 16
 
@@ -60,7 +61,7 @@
     int cluster_sectors;
     int l2_bits;
     int l2_size;
-    int l1_size;
+    unsigned int l1_size;
     uint64_t cluster_offset_mask;
     uint64_t l1_table_offset;
     uint64_t *l1_table;
@@ -96,7 +97,8 @@
                      Error **errp)
 {
     BDRVQcowState *s = bs->opaque;
-    int len, i, shift, ret;
+    unsigned int len, i, shift;
+    int ret;
     QCowHeader header;
 
     ret = bdrv_pread(bs->file, 0, &header, sizeof(header));
@@ -127,11 +129,25 @@
         goto fail;
     }
 
-    if (header.size <= 1 || header.cluster_bits < 9) {
-        error_setg(errp, "invalid value in qcow header");
+    if (header.size <= 1) {
+        error_setg(errp, "Image size is too small (must be at least 2 bytes)");
         ret = -EINVAL;
         goto fail;
     }
+    if (header.cluster_bits < 9 || header.cluster_bits > 16) {
+        error_setg(errp, "Cluster size must be between 512 and 64k");
+        ret = -EINVAL;
+        goto fail;
+    }
+
+    /* l2_bits specifies number of entries; storing a uint64_t in each entry,
+     * so bytes = num_entries << 3. */
+    if (header.l2_bits < 9 - 3 || header.l2_bits > 16 - 3) {
+        error_setg(errp, "L2 table size must be between 512 and 64k");
+        ret = -EINVAL;
+        goto fail;
+    }
+
     if (header.crypt_method > QCOW_CRYPT_AES) {
         error_setg(errp, "invalid encryption method in qcow header");
         ret = -EINVAL;
@@ -151,7 +167,19 @@
 
     /* read the level 1 table */
     shift = s->cluster_bits + s->l2_bits;
-    s->l1_size = (header.size + (1LL << shift) - 1) >> shift;
+    if (header.size > UINT64_MAX - (1LL << shift)) {
+        error_setg(errp, "Image too large");
+        ret = -EINVAL;
+        goto fail;
+    } else {
+        uint64_t l1_size = (header.size + (1LL << shift) - 1) >> shift;
+        if (l1_size > INT_MAX / sizeof(uint64_t)) {
+            error_setg(errp, "Image too large");
+            ret = -EINVAL;
+            goto fail;
+        }
+        s->l1_size = l1_size;
+    }
 
     s->l1_table_offset = header.l1_table_offset;
     s->l1_table = g_malloc(s->l1_size * sizeof(uint64_t));
@@ -175,7 +203,9 @@
     if (header.backing_file_offset != 0) {
         len = header.backing_file_size;
         if (len > 1023) {
-            len = 1023;
+            error_setg(errp, "Backing file name too long");
+            ret = -EINVAL;
+            goto fail;
         }
         ret = bdrv_pread(bs->file, header.backing_file_offset,
                    bs->backing_file, len);
diff --git a/block/qcow2-cluster.c b/block/qcow2-cluster.c
index 76d2bcf..4208dc0 100644
--- a/block/qcow2-cluster.c
+++ b/block/qcow2-cluster.c
@@ -379,7 +379,8 @@
     BLKDBG_EVENT(bs->file, BLKDBG_COW_READ);
 
     if (!bs->drv) {
-        return -ENOMEDIUM;
+        ret = -ENOMEDIUM;
+        goto out;
     }
 
     /* Call .bdrv_co_readv() directly instead of using the public block-layer
diff --git a/block/qcow2.c b/block/qcow2.c
index a4b97e8..a54d2ba 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -1308,6 +1308,7 @@
     options = qdict_clone_shallow(bs->options);
 
     ret = qcow2_open(bs, options, flags, &local_err);
+    QDECREF(options);
     if (local_err) {
         error_setg(errp, "Could not reopen qcow2 layer: %s",
                    error_get_pretty(local_err));
@@ -1318,8 +1319,6 @@
         return;
     }
 
-    QDECREF(options);
-
     if (crypt_method) {
         s->crypt_method = crypt_method;
         memcpy(&s->aes_encrypt_key, &aes_encrypt_key, sizeof(aes_encrypt_key));
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 6586a0c..b7f0f26 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -1192,7 +1192,7 @@
         if (size == 0)
 #endif
 #if defined(__APPLE__) && defined(__MACH__)
-        size = LONG_LONG_MAX;
+        size = LLONG_MAX;
 #else
         size = lseek(fd, 0LL, SEEK_END);
 #endif
diff --git a/block/rbd.c b/block/rbd.c
index dbc79f4..09af484 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -105,7 +105,7 @@
 static int qemu_rbd_next_tok(char *dst, int dst_len,
                              char *src, char delim,
                              const char *name,
-                             char **p)
+                             char **p, Error **errp)
 {
     int l;
     char *end;
@@ -128,10 +128,10 @@
     }
     l = strlen(src);
     if (l >= dst_len) {
-        error_report("%s too long", name);
+        error_setg(errp, "%s too long", name);
         return -EINVAL;
     } else if (l == 0) {
-        error_report("%s too short", name);
+        error_setg(errp, "%s too short", name);
         return -EINVAL;
     }
 
@@ -157,13 +157,15 @@
                               char *pool, int pool_len,
                               char *snap, int snap_len,
                               char *name, int name_len,
-                              char *conf, int conf_len)
+                              char *conf, int conf_len,
+                              Error **errp)
 {
     const char *start;
     char *p, *buf;
     int ret;
 
     if (!strstart(filename, "rbd:", &start)) {
+        error_setg(errp, "File name must start with 'rbd:'");
         return -EINVAL;
     }
 
@@ -172,7 +174,8 @@
     *snap = '\0';
     *conf = '\0';
 
-    ret = qemu_rbd_next_tok(pool, pool_len, p, '/', "pool name", &p);
+    ret = qemu_rbd_next_tok(pool, pool_len, p,
+                            '/', "pool name", &p, errp);
     if (ret < 0 || !p) {
         ret = -EINVAL;
         goto done;
@@ -180,21 +183,25 @@
     qemu_rbd_unescape(pool);
 
     if (strchr(p, '@')) {
-        ret = qemu_rbd_next_tok(name, name_len, p, '@', "object name", &p);
+        ret = qemu_rbd_next_tok(name, name_len, p,
+                                '@', "object name", &p, errp);
         if (ret < 0) {
             goto done;
         }
-        ret = qemu_rbd_next_tok(snap, snap_len, p, ':', "snap name", &p);
+        ret = qemu_rbd_next_tok(snap, snap_len, p,
+                                ':', "snap name", &p, errp);
         qemu_rbd_unescape(snap);
     } else {
-        ret = qemu_rbd_next_tok(name, name_len, p, ':', "object name", &p);
+        ret = qemu_rbd_next_tok(name, name_len, p,
+                                ':', "object name", &p, errp);
     }
     qemu_rbd_unescape(name);
     if (ret < 0 || !p) {
         goto done;
     }
 
-    ret = qemu_rbd_next_tok(conf, conf_len, p, '\0', "configuration", &p);
+    ret = qemu_rbd_next_tok(conf, conf_len, p,
+                            '\0', "configuration", &p, errp);
 
 done:
     g_free(buf);
@@ -229,7 +236,7 @@
     return NULL;
 }
 
-static int qemu_rbd_set_conf(rados_t cluster, const char *conf)
+static int qemu_rbd_set_conf(rados_t cluster, const char *conf, Error **errp)
 {
     char *p, *buf;
     char name[RBD_MAX_CONF_NAME_SIZE];
@@ -241,20 +248,20 @@
 
     while (p) {
         ret = qemu_rbd_next_tok(name, sizeof(name), p,
-                                '=', "conf option name", &p);
+                                '=', "conf option name", &p, errp);
         if (ret < 0) {
             break;
         }
         qemu_rbd_unescape(name);
 
         if (!p) {
-            error_report("conf option %s has no value", name);
+            error_setg(errp, "conf option %s has no value", name);
             ret = -EINVAL;
             break;
         }
 
         ret = qemu_rbd_next_tok(value, sizeof(value), p,
-                                ':', "conf option value", &p);
+                                ':', "conf option value", &p, errp);
         if (ret < 0) {
             break;
         }
@@ -263,7 +270,7 @@
         if (strcmp(name, "conf") == 0) {
             ret = rados_conf_read_file(cluster, value);
             if (ret < 0) {
-                error_report("error reading conf file %s", value);
+                error_setg(errp, "error reading conf file %s", value);
                 break;
             }
         } else if (strcmp(name, "id") == 0) {
@@ -271,7 +278,7 @@
         } else {
             ret = rados_conf_set(cluster, name, value);
             if (ret < 0) {
-                error_report("invalid conf option %s", name);
+                error_setg(errp, "invalid conf option %s", name);
                 ret = -EINVAL;
                 break;
             }
@@ -285,6 +292,7 @@
 static int qemu_rbd_create(const char *filename, QEMUOptionParameter *options,
                            Error **errp)
 {
+    Error *local_err = NULL;
     int64_t bytes = 0;
     int64_t objsize;
     int obj_order = 0;
@@ -301,7 +309,8 @@
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
                            name, sizeof(name),
-                           conf, sizeof(conf)) < 0) {
+                           conf, sizeof(conf), &local_err) < 0) {
+        error_propagate(errp, local_err);
         return -EINVAL;
     }
 
@@ -313,11 +322,11 @@
             if (options->value.n) {
                 objsize = options->value.n;
                 if ((objsize - 1) & objsize) {    /* not a power of 2? */
-                    error_report("obj size needs to be power of 2");
+                    error_setg(errp, "obj size needs to be power of 2");
                     return -EINVAL;
                 }
                 if (objsize < 4096) {
-                    error_report("obj size too small");
+                    error_setg(errp, "obj size too small");
                     return -EINVAL;
                 }
                 obj_order = ffs(objsize) - 1;
@@ -328,7 +337,7 @@
 
     clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
     if (rados_create(&cluster, clientname) < 0) {
-        error_report("error initializing");
+        error_setg(errp, "error initializing");
         return -EIO;
     }
 
@@ -338,20 +347,20 @@
     }
 
     if (conf[0] != '\0' &&
-        qemu_rbd_set_conf(cluster, conf) < 0) {
-        error_report("error setting config options");
+        qemu_rbd_set_conf(cluster, conf, &local_err) < 0) {
         rados_shutdown(cluster);
+        error_propagate(errp, local_err);
         return -EIO;
     }
 
     if (rados_connect(cluster) < 0) {
-        error_report("error connecting");
+        error_setg(errp, "error connecting");
         rados_shutdown(cluster);
         return -EIO;
     }
 
     if (rados_ioctx_create(cluster, pool, &io_ctx) < 0) {
-        error_report("error opening pool %s", pool);
+        error_setg(errp, "error opening pool %s", pool);
         rados_shutdown(cluster);
         return -EIO;
     }
@@ -441,8 +450,7 @@
     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         qemu_opts_del(opts);
         return -EINVAL;
     }
@@ -452,7 +460,7 @@
     if (qemu_rbd_parsename(filename, pool, sizeof(pool),
                            snap_buf, sizeof(snap_buf),
                            s->name, sizeof(s->name),
-                           conf, sizeof(conf)) < 0) {
+                           conf, sizeof(conf), errp) < 0) {
         r = -EINVAL;
         goto failed_opts;
     }
@@ -460,7 +468,7 @@
     clientname = qemu_rbd_parse_clientname(conf, clientname_buf);
     r = rados_create(&s->cluster, clientname);
     if (r < 0) {
-        error_report("error initializing");
+        error_setg(&local_err, "error initializing");
         goto failed_opts;
     }
 
@@ -488,28 +496,27 @@
     }
 
     if (conf[0] != '\0') {
-        r = qemu_rbd_set_conf(s->cluster, conf);
+        r = qemu_rbd_set_conf(s->cluster, conf, errp);
         if (r < 0) {
-            error_report("error setting config options");
             goto failed_shutdown;
         }
     }
 
     r = rados_connect(s->cluster);
     if (r < 0) {
-        error_report("error connecting");
+        error_setg(&local_err, "error connecting");
         goto failed_shutdown;
     }
 
     r = rados_ioctx_create(s->cluster, pool, &s->io_ctx);
     if (r < 0) {
-        error_report("error opening pool %s", pool);
+        error_setg(&local_err, "error opening pool %s", pool);
         goto failed_shutdown;
     }
 
     r = rbd_open(s->io_ctx, s->name, &s->image, s->snap);
     if (r < 0) {
-        error_report("error reading header from %s", s->name);
+        error_setg(&local_err, "error reading header from %s", s->name);
         goto failed_open;
     }
 
diff --git a/block/sheepdog.c b/block/sheepdog.c
index 2c3fb01..4ecbf5f 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -526,17 +526,16 @@
     return acb;
 }
 
-static int connect_to_sdog(BDRVSheepdogState *s)
+static int connect_to_sdog(BDRVSheepdogState *s, Error **errp)
 {
     int fd;
-    Error *err = NULL;
 
     if (s->is_unix) {
-        fd = unix_connect(s->host_spec, &err);
+        fd = unix_connect(s->host_spec, errp);
     } else {
-        fd = inet_connect(s->host_spec, &err);
+        fd = inet_connect(s->host_spec, errp);
 
-        if (err == NULL) {
+        if (fd >= 0) {
             int ret = socket_set_nodelay(fd);
             if (ret < 0) {
                 error_report("%s", strerror(errno));
@@ -544,10 +543,7 @@
         }
     }
 
-    if (err != NULL) {
-        qerror_report_err(err);
-        error_free(err);
-    } else {
+    if (fd >= 0) {
         qemu_set_nonblock(fd);
     }
 
@@ -672,7 +668,7 @@
                            enum AIOCBState aiocb_type);
 static void coroutine_fn resend_aioreq(BDRVSheepdogState *s, AIOReq *aio_req);
 static int reload_inode(BDRVSheepdogState *s, uint32_t snapid, const char *tag);
-static int get_sheep_fd(BDRVSheepdogState *s);
+static int get_sheep_fd(BDRVSheepdogState *s, Error **errp);
 static void co_write_request(void *opaque);
 
 static AIOReq *find_pending_req(BDRVSheepdogState *s, uint64_t oid)
@@ -709,6 +705,7 @@
 
 static coroutine_fn void reconnect_to_sdog(void *opaque)
 {
+    Error *local_err = NULL;
     BDRVSheepdogState *s = opaque;
     AIOReq *aio_req, *next;
 
@@ -723,9 +720,11 @@
 
     /* Try to reconnect the sheepdog server every one second. */
     while (s->fd < 0) {
-        s->fd = get_sheep_fd(s);
+        s->fd = get_sheep_fd(s, &local_err);
         if (s->fd < 0) {
             DPRINTF("Wait for connection to be established\n");
+            error_report("%s", error_get_pretty(local_err));
+            error_free(local_err);
             co_aio_sleep_ns(bdrv_get_aio_context(s->bs), QEMU_CLOCK_REALTIME,
                             1000000000ULL);
         }
@@ -914,11 +913,11 @@
  * We cannot use this descriptor for other operations because
  * the block driver may be on waiting response from the server.
  */
-static int get_sheep_fd(BDRVSheepdogState *s)
+static int get_sheep_fd(BDRVSheepdogState *s, Error **errp)
 {
     int fd;
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, errp);
     if (fd < 0) {
         return fd;
     }
@@ -1061,7 +1060,7 @@
 
 static int find_vdi_name(BDRVSheepdogState *s, const char *filename,
                          uint32_t snapid, const char *tag, uint32_t *vid,
-                         bool lock)
+                         bool lock, Error **errp)
 {
     int ret, fd;
     SheepdogVdiReq hdr;
@@ -1069,7 +1068,7 @@
     unsigned int wlen, rlen = 0;
     char buf[SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN];
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, errp);
     if (fd < 0) {
         return fd;
     }
@@ -1095,12 +1094,13 @@
 
     ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
     if (ret) {
+        error_setg_errno(errp, -ret, "cannot get vdi info");
         goto out;
     }
 
     if (rsp->result != SD_RES_SUCCESS) {
-        error_report("cannot get vdi info, %s, %s %" PRIu32 " %s",
-                     sd_strerror(rsp->result), filename, snapid, tag);
+        error_setg(errp, "cannot get vdi info, %s, %s %" PRIu32 " %s",
+                   sd_strerror(rsp->result), filename, snapid, tag);
         if (rsp->result == SD_RES_NO_VDI) {
             ret = -ENOENT;
         } else {
@@ -1263,19 +1263,24 @@
 /* update inode with the latest state */
 static int reload_inode(BDRVSheepdogState *s, uint32_t snapid, const char *tag)
 {
+    Error *local_err = NULL;
     SheepdogInode *inode;
     int ret = 0, fd;
     uint32_t vid = 0;
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         return -EIO;
     }
 
     inode = g_malloc(sizeof(s->inode));
 
-    ret = find_vdi_name(s, s->name, snapid, tag, &vid, false);
+    ret = find_vdi_name(s, s->name, snapid, tag, &vid, false, &local_err);
     if (ret) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         goto out;
     }
 
@@ -1386,8 +1391,7 @@
     opts = qemu_opts_create(&runtime_opts, NULL, 0, &error_abort);
     qemu_opts_absorb_qdict(opts, options, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
-        error_free(local_err);
+        error_propagate(errp, local_err);
         ret = -EINVAL;
         goto out;
     }
@@ -1408,15 +1412,16 @@
         ret = parse_vdiname(s, filename, vdi, &snapid, tag);
     }
     if (ret < 0) {
+        error_setg(errp, "Can't parse filename");
         goto out;
     }
-    s->fd = get_sheep_fd(s);
+    s->fd = get_sheep_fd(s, errp);
     if (s->fd < 0) {
         ret = s->fd;
         goto out;
     }
 
-    ret = find_vdi_name(s, vdi, snapid, tag, &vid, true);
+    ret = find_vdi_name(s, vdi, snapid, tag, &vid, true, errp);
     if (ret) {
         goto out;
     }
@@ -1436,7 +1441,7 @@
         s->is_snapshot = true;
     }
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, errp);
     if (fd < 0) {
         ret = fd;
         goto out;
@@ -1449,6 +1454,7 @@
     closesocket(fd);
 
     if (ret) {
+        error_setg(errp, "Can't read snapshot inode");
         goto out;
     }
 
@@ -1472,7 +1478,8 @@
     return ret;
 }
 
-static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot)
+static int do_sd_create(BDRVSheepdogState *s, uint32_t *vdi_id, int snapshot,
+                        Error **errp)
 {
     SheepdogVdiReq hdr;
     SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
@@ -1480,7 +1487,7 @@
     unsigned int wlen, rlen = 0;
     char buf[SD_MAX_VDI_LEN];
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, errp);
     if (fd < 0) {
         return fd;
     }
@@ -1510,11 +1517,12 @@
     closesocket(fd);
 
     if (ret) {
+        error_setg_errno(errp, -ret, "create failed");
         return ret;
     }
 
     if (rsp->result != SD_RES_SUCCESS) {
-        error_report("%s, %s", sd_strerror(rsp->result), s->inode.name);
+        error_setg(errp, "%s, %s", sd_strerror(rsp->result), s->inode.name);
         return -EIO;
     }
 
@@ -1525,21 +1533,18 @@
     return 0;
 }
 
-static int sd_prealloc(const char *filename)
+static int sd_prealloc(const char *filename, Error **errp)
 {
     BlockDriverState *bs = NULL;
     uint32_t idx, max_idx;
     int64_t vdi_size;
     void *buf = g_malloc0(SD_DATA_OBJ_SIZE);
-    Error *local_err = NULL;
     int ret;
 
     ret = bdrv_open(&bs, filename, NULL, NULL, BDRV_O_RDWR | BDRV_O_PROTOCOL,
-                    NULL, &local_err);
+                    NULL, errp);
     if (ret < 0) {
-        qerror_report_err(local_err);
-        error_free(local_err);
-        goto out;
+        goto out_with_err_set;
     }
 
     vdi_size = bdrv_getlength(bs);
@@ -1563,7 +1568,12 @@
             goto out;
         }
     }
+
 out:
+    if (ret < 0) {
+        error_setg_errno(errp, -ret, "Can't pre-allocate");
+    }
+out_with_err_set:
     if (bs) {
         bdrv_unref(bs);
     }
@@ -1636,7 +1646,6 @@
     char tag[SD_MAX_VDI_TAG_LEN];
     uint32_t snapid;
     bool prealloc = false;
-    Error *local_err = NULL;
 
     s = g_malloc0(sizeof(BDRVSheepdogState));
 
@@ -1647,6 +1656,7 @@
         ret = parse_vdiname(s, filename, s->name, &snapid, tag);
     }
     if (ret < 0) {
+        error_setg(errp, "Can't parse filename");
         goto out;
     }
 
@@ -1661,8 +1671,8 @@
             } else if (!strcmp(options->value.s, "full")) {
                 prealloc = true;
             } else {
-                error_report("Invalid preallocation mode: '%s'",
-                             options->value.s);
+                error_setg(errp, "Invalid preallocation mode: '%s'",
+                           options->value.s);
                 ret = -EINVAL;
                 goto out;
             }
@@ -1670,6 +1680,8 @@
             if (options->value.s) {
                 ret = parse_redundancy(s, options->value.s);
                 if (ret < 0) {
+                    error_setg(errp, "Invalid redundancy mode: '%s'",
+                               options->value.s);
                     goto out;
                 }
             }
@@ -1678,7 +1690,7 @@
     }
 
     if (s->inode.vdi_size > SD_MAX_VDI_SIZE) {
-        error_report("too big image size");
+        error_setg(errp, "too big image size");
         ret = -EINVAL;
         goto out;
     }
@@ -1691,24 +1703,22 @@
         /* Currently, only Sheepdog backing image is supported. */
         drv = bdrv_find_protocol(backing_file, true);
         if (!drv || strcmp(drv->protocol_name, "sheepdog") != 0) {
-            error_report("backing_file must be a sheepdog image");
+            error_setg(errp, "backing_file must be a sheepdog image");
             ret = -EINVAL;
             goto out;
         }
 
         bs = NULL;
         ret = bdrv_open(&bs, backing_file, NULL, NULL, BDRV_O_PROTOCOL, NULL,
-                        &local_err);
+                        errp);
         if (ret < 0) {
-            qerror_report_err(local_err);
-            error_free(local_err);
             goto out;
         }
 
         base = bs->opaque;
 
         if (!is_snapshot(&base->inode)) {
-            error_report("cannot clone from a non snapshot vdi");
+            error_setg(errp, "cannot clone from a non snapshot vdi");
             bdrv_unref(bs);
             ret = -EINVAL;
             goto out;
@@ -1717,12 +1727,14 @@
         bdrv_unref(bs);
     }
 
-    ret = do_sd_create(s, &vid, 0);
-    if (!prealloc || ret) {
+    ret = do_sd_create(s, &vid, 0, errp);
+    if (ret) {
         goto out;
     }
 
-    ret = sd_prealloc(filename);
+    if (prealloc) {
+        ret = sd_prealloc(filename, errp);
+    }
 out:
     g_free(s);
     return ret;
@@ -1730,6 +1742,7 @@
 
 static void sd_close(BlockDriverState *bs)
 {
+    Error *local_err = NULL;
     BDRVSheepdogState *s = bs->opaque;
     SheepdogVdiReq hdr;
     SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
@@ -1738,8 +1751,10 @@
 
     DPRINTF("%s\n", s->name);
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         return;
     }
 
@@ -1774,6 +1789,7 @@
 
 static int sd_truncate(BlockDriverState *bs, int64_t offset)
 {
+    Error *local_err = NULL;
     BDRVSheepdogState *s = bs->opaque;
     int ret, fd;
     unsigned int datalen;
@@ -1786,8 +1802,10 @@
         return -EINVAL;
     }
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         return fd;
     }
 
@@ -1846,6 +1864,7 @@
 /* Delete current working VDI on the snapshot chain */
 static bool sd_delete(BDRVSheepdogState *s)
 {
+    Error *local_err = NULL;
     unsigned int wlen = SD_MAX_VDI_LEN, rlen = 0;
     SheepdogVdiReq hdr = {
         .opcode = SD_OP_DEL_VDI,
@@ -1856,8 +1875,10 @@
     SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
     int fd, ret;
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         return false;
     }
 
@@ -1885,6 +1906,7 @@
  */
 static int sd_create_branch(BDRVSheepdogState *s)
 {
+    Error *local_err = NULL;
     int ret, fd;
     uint32_t vid;
     char *buf;
@@ -1900,15 +1922,19 @@
      * false bail out.
      */
     deleted = sd_delete(s);
-    ret = do_sd_create(s, &vid, !deleted);
+    ret = do_sd_create(s, &vid, !deleted, &local_err);
     if (ret) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         goto out;
     }
 
     DPRINTF("%" PRIx32 " is created.\n", vid);
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         ret = fd;
         goto out;
     }
@@ -2122,6 +2148,7 @@
 
 static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 {
+    Error *local_err = NULL;
     BDRVSheepdogState *s = bs->opaque;
     int ret, fd;
     uint32_t new_vid;
@@ -2149,10 +2176,13 @@
     strncpy(s->inode.tag, sn_info->name, sizeof(s->inode.tag));
     /* we don't need to update entire object */
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
+    inode = g_malloc(datalen);
 
     /* refresh inode. */
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         ret = fd;
         goto cleanup;
     }
@@ -2164,15 +2194,15 @@
         goto cleanup;
     }
 
-    ret = do_sd_create(s, &new_vid, 1);
+    ret = do_sd_create(s, &new_vid, 1, &local_err);
     if (ret < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         error_report("failed to create inode for snapshot. %s",
                      strerror(errno));
         goto cleanup;
     }
 
-    inode = (SheepdogInode *)g_malloc(datalen);
-
     ret = read_object(fd, (char *)inode, vid_to_vdi_oid(new_vid),
                       s->inode.nr_copies, datalen, 0, s->cache_flags);
 
@@ -2186,6 +2216,7 @@
             s->inode.name, s->inode.snap_id, s->inode.vdi_id);
 
 cleanup:
+    g_free(inode);
     closesocket(fd);
     return ret;
 }
@@ -2249,6 +2280,7 @@
 
 static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 {
+    Error *local_err = NULL;
     BDRVSheepdogState *s = bs->opaque;
     SheepdogReq req;
     int fd, nr = 1024, ret, max = BITS_TO_LONGS(SD_NR_VDIS) * sizeof(long);
@@ -2263,8 +2295,10 @@
 
     vdi_inuse = g_malloc(max);
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         ret = fd;
         goto out;
     }
@@ -2290,8 +2324,10 @@
     hval = fnv_64a_buf(s->name, strlen(s->name), FNV1A_64_INIT);
     start_nr = hval & (SD_NR_VDIS - 1);
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         ret = fd;
         goto out;
     }
@@ -2341,6 +2377,7 @@
 static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
                                 int64_t pos, int size, int load)
 {
+    Error *local_err = NULL;
     bool create;
     int fd, ret = 0, remaining = size;
     unsigned int data_len;
@@ -2349,8 +2386,10 @@
     uint32_t vdi_index;
     uint32_t vdi_id = load ? s->inode.parent_vdi_id : s->inode.vdi_id;
 
-    fd = connect_to_sdog(s);
+    fd = connect_to_sdog(s, &local_err);
     if (fd < 0) {
+        error_report("%s", error_get_pretty(local_err));;
+        error_free(local_err);
         return fd;
     }
 
diff --git a/block/ssh.c b/block/ssh.c
index aa63c9d..b212971 100644
--- a/block/ssh.c
+++ b/block/ssh.c
@@ -106,30 +106,59 @@
     }
 }
 
-/* Wrappers around error_report which make sure to dump as much
- * information from libssh2 as possible.
- */
-static void GCC_FMT_ATTR(2, 3)
-session_error_report(BDRVSSHState *s, const char *fs, ...)
+static void GCC_FMT_ATTR(3, 4)
+session_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...)
 {
     va_list args;
+    char *msg;
 
     va_start(args, fs);
-    error_vprintf(fs, args);
+    msg = g_strdup_vprintf(fs, args);
+    va_end(args);
 
-    if ((s)->session) {
+    if (s->session) {
         char *ssh_err;
         int ssh_err_code;
 
-        libssh2_session_last_error((s)->session, &ssh_err, NULL, 0);
         /* This is not an errno.  See <libssh2.h>. */
-        ssh_err_code = libssh2_session_last_errno((s)->session);
-
-        error_printf(": %s (libssh2 error code: %d)", ssh_err, ssh_err_code);
+        ssh_err_code = libssh2_session_last_error(s->session,
+                                                  &ssh_err, NULL, 0);
+        error_setg(errp, "%s: %s (libssh2 error code: %d)",
+                   msg, ssh_err, ssh_err_code);
+    } else {
+        error_setg(errp, "%s", msg);
     }
+    g_free(msg);
+}
 
+static void GCC_FMT_ATTR(3, 4)
+sftp_error_setg(Error **errp, BDRVSSHState *s, const char *fs, ...)
+{
+    va_list args;
+    char *msg;
+
+    va_start(args, fs);
+    msg = g_strdup_vprintf(fs, args);
     va_end(args);
-    error_printf("\n");
+
+    if (s->sftp) {
+        char *ssh_err;
+        int ssh_err_code;
+        unsigned long sftp_err_code;
+
+        /* This is not an errno.  See <libssh2.h>. */
+        ssh_err_code = libssh2_session_last_error(s->session,
+                                                  &ssh_err, NULL, 0);
+        /* See <libssh2_sftp.h>. */
+        sftp_err_code = libssh2_sftp_last_error((s)->sftp);
+
+        error_setg(errp,
+                   "%s: %s (libssh2 error code: %d, sftp error code: %lu)",
+                   msg, ssh_err, ssh_err_code, sftp_err_code);
+    } else {
+        error_setg(errp, "%s", msg);
+    }
+    g_free(msg);
 }
 
 static void GCC_FMT_ATTR(2, 3)
@@ -145,9 +174,9 @@
         int ssh_err_code;
         unsigned long sftp_err_code;
 
-        libssh2_session_last_error((s)->session, &ssh_err, NULL, 0);
         /* This is not an errno.  See <libssh2.h>. */
-        ssh_err_code = libssh2_session_last_errno((s)->session);
+        ssh_err_code = libssh2_session_last_error(s->session,
+                                                  &ssh_err, NULL, 0);
         /* See <libssh2_sftp.h>. */
         sftp_err_code = libssh2_sftp_last_error((s)->sftp);
 
@@ -243,7 +272,7 @@
 }
 
 static int check_host_key_knownhosts(BDRVSSHState *s,
-                                     const char *host, int port)
+                                     const char *host, int port, Error **errp)
 {
     const char *home;
     char *knh_file = NULL;
@@ -257,14 +286,15 @@
     hostkey = libssh2_session_hostkey(s->session, &len, &type);
     if (!hostkey) {
         ret = -EINVAL;
-        session_error_report(s, "failed to read remote host key");
+        session_error_setg(errp, s, "failed to read remote host key");
         goto out;
     }
 
     knh = libssh2_knownhost_init(s->session);
     if (!knh) {
         ret = -EINVAL;
-        session_error_report(s, "failed to initialize known hosts support");
+        session_error_setg(errp, s,
+                           "failed to initialize known hosts support");
         goto out;
     }
 
@@ -289,21 +319,23 @@
         break;
     case LIBSSH2_KNOWNHOST_CHECK_MISMATCH:
         ret = -EINVAL;
-        session_error_report(s, "host key does not match the one in known_hosts (found key %s)",
-                             found->key);
+        session_error_setg(errp, s,
+                      "host key does not match the one in known_hosts"
+                      " (found key %s)", found->key);
         goto out;
     case LIBSSH2_KNOWNHOST_CHECK_NOTFOUND:
         ret = -EINVAL;
-        session_error_report(s, "no host key was found in known_hosts");
+        session_error_setg(errp, s, "no host key was found in known_hosts");
         goto out;
     case LIBSSH2_KNOWNHOST_CHECK_FAILURE:
         ret = -EINVAL;
-        session_error_report(s, "failure matching the host key with known_hosts");
+        session_error_setg(errp, s,
+                      "failure matching the host key with known_hosts");
         goto out;
     default:
         ret = -EINVAL;
-        session_error_report(s, "unknown error matching the host key with known_hosts (%d)",
-                             r);
+        session_error_setg(errp, s, "unknown error matching the host key"
+                      " with known_hosts (%d)", r);
         goto out;
     }
 
@@ -358,20 +390,20 @@
 
 static int
 check_host_key_hash(BDRVSSHState *s, const char *hash,
-                    int hash_type, size_t fingerprint_len)
+                    int hash_type, size_t fingerprint_len, Error **errp)
 {
     const char *fingerprint;
 
     fingerprint = libssh2_hostkey_hash(s->session, hash_type);
     if (!fingerprint) {
-        session_error_report(s, "failed to read remote host key");
+        session_error_setg(errp, s, "failed to read remote host key");
         return -EINVAL;
     }
 
     if(compare_fingerprint((unsigned char *) fingerprint, fingerprint_len,
                            hash) != 0) {
-        error_report("remote host key does not match host_key_check '%s'",
-                     hash);
+        error_setg(errp, "remote host key does not match host_key_check '%s'",
+                   hash);
         return -EPERM;
     }
 
@@ -379,7 +411,7 @@
 }
 
 static int check_host_key(BDRVSSHState *s, const char *host, int port,
-                          const char *host_key_check)
+                          const char *host_key_check, Error **errp)
 {
     /* host_key_check=no */
     if (strcmp(host_key_check, "no") == 0) {
@@ -389,25 +421,25 @@
     /* host_key_check=md5:xx:yy:zz:... */
     if (strncmp(host_key_check, "md5:", 4) == 0) {
         return check_host_key_hash(s, &host_key_check[4],
-                                   LIBSSH2_HOSTKEY_HASH_MD5, 16);
+                                   LIBSSH2_HOSTKEY_HASH_MD5, 16, errp);
     }
 
     /* host_key_check=sha1:xx:yy:zz:... */
     if (strncmp(host_key_check, "sha1:", 5) == 0) {
         return check_host_key_hash(s, &host_key_check[5],
-                                   LIBSSH2_HOSTKEY_HASH_SHA1, 20);
+                                   LIBSSH2_HOSTKEY_HASH_SHA1, 20, errp);
     }
 
     /* host_key_check=yes */
     if (strcmp(host_key_check, "yes") == 0) {
-        return check_host_key_knownhosts(s, host, port);
+        return check_host_key_knownhosts(s, host, port, errp);
     }
 
-    error_report("unknown host_key_check setting (%s)", host_key_check);
+    error_setg(errp, "unknown host_key_check setting (%s)", host_key_check);
     return -EINVAL;
 }
 
-static int authenticate(BDRVSSHState *s, const char *user)
+static int authenticate(BDRVSSHState *s, const char *user, Error **errp)
 {
     int r, ret;
     const char *userauthlist;
@@ -418,7 +450,8 @@
     userauthlist = libssh2_userauth_list(s->session, user, strlen(user));
     if (strstr(userauthlist, "publickey") == NULL) {
         ret = -EPERM;
-        error_report("remote server does not support \"publickey\" authentication");
+        error_setg(errp,
+                "remote server does not support \"publickey\" authentication");
         goto out;
     }
 
@@ -426,17 +459,18 @@
     agent = libssh2_agent_init(s->session);
     if (!agent) {
         ret = -EINVAL;
-        session_error_report(s, "failed to initialize ssh-agent support");
+        session_error_setg(errp, s, "failed to initialize ssh-agent support");
         goto out;
     }
     if (libssh2_agent_connect(agent)) {
         ret = -ECONNREFUSED;
-        session_error_report(s, "failed to connect to ssh-agent");
+        session_error_setg(errp, s, "failed to connect to ssh-agent");
         goto out;
     }
     if (libssh2_agent_list_identities(agent)) {
         ret = -EINVAL;
-        session_error_report(s, "failed requesting identities from ssh-agent");
+        session_error_setg(errp, s,
+                           "failed requesting identities from ssh-agent");
         goto out;
     }
 
@@ -447,7 +481,8 @@
         }
         if (r < 0) {
             ret = -EINVAL;
-            session_error_report(s, "failed to obtain identity from ssh-agent");
+            session_error_setg(errp, s,
+                               "failed to obtain identity from ssh-agent");
             goto out;
         }
         r = libssh2_agent_userauth(agent, user, identity);
@@ -461,8 +496,8 @@
     }
 
     ret = -EPERM;
-    error_report("failed to authenticate using publickey authentication "
-                 "and the identities held by your ssh-agent");
+    error_setg(errp, "failed to authenticate using publickey authentication "
+               "and the identities held by your ssh-agent");
 
  out:
     if (agent != NULL) {
@@ -476,10 +511,9 @@
 }
 
 static int connect_to_ssh(BDRVSSHState *s, QDict *options,
-                          int ssh_flags, int creat_mode)
+                          int ssh_flags, int creat_mode, Error **errp)
 {
     int r, ret;
-    Error *err = NULL;
     const char *host, *user, *path, *host_key_check;
     int port;
 
@@ -498,6 +532,7 @@
     } else {
         user = g_get_user_name();
         if (!user) {
+            error_setg_errno(errp, errno, "Can't get user name");
             ret = -errno;
             goto err;
         }
@@ -514,11 +549,9 @@
     s->hostport = g_strdup_printf("%s:%d", host, port);
 
     /* Open the socket and connect. */
-    s->sock = inet_connect(s->hostport, &err);
-    if (err != NULL) {
+    s->sock = inet_connect(s->hostport, errp);
+    if (s->sock < 0) {
         ret = -errno;
-        qerror_report_err(err);
-        error_free(err);
         goto err;
     }
 
@@ -526,7 +559,7 @@
     s->session = libssh2_session_init();
     if (!s->session) {
         ret = -EINVAL;
-        session_error_report(s, "failed to initialize libssh2 session");
+        session_error_setg(errp, s, "failed to initialize libssh2 session");
         goto err;
     }
 
@@ -537,18 +570,18 @@
     r = libssh2_session_handshake(s->session, s->sock);
     if (r != 0) {
         ret = -EINVAL;
-        session_error_report(s, "failed to establish SSH session");
+        session_error_setg(errp, s, "failed to establish SSH session");
         goto err;
     }
 
     /* Check the remote host's key against known_hosts. */
-    ret = check_host_key(s, host, port, host_key_check);
+    ret = check_host_key(s, host, port, host_key_check, errp);
     if (ret < 0) {
         goto err;
     }
 
     /* Authenticate. */
-    ret = authenticate(s, user);
+    ret = authenticate(s, user, errp);
     if (ret < 0) {
         goto err;
     }
@@ -556,7 +589,7 @@
     /* Start SFTP. */
     s->sftp = libssh2_sftp_init(s->session);
     if (!s->sftp) {
-        session_error_report(s, "failed to initialize sftp handle");
+        session_error_setg(errp, s, "failed to initialize sftp handle");
         ret = -EINVAL;
         goto err;
     }
@@ -566,14 +599,14 @@
             path, ssh_flags, creat_mode);
     s->sftp_handle = libssh2_sftp_open(s->sftp, path, ssh_flags, creat_mode);
     if (!s->sftp_handle) {
-        session_error_report(s, "failed to open remote file '%s'", path);
+        session_error_setg(errp, s, "failed to open remote file '%s'", path);
         ret = -EINVAL;
         goto err;
     }
 
     r = libssh2_sftp_fstat(s->sftp_handle, &s->attrs);
     if (r < 0) {
-        sftp_error_report(s, "failed to read file attributes");
+        sftp_error_setg(errp, s, "failed to read file attributes");
         return -EINVAL;
     }
 
@@ -623,7 +656,7 @@
     }
 
     /* Start up SSH. */
-    ret = connect_to_ssh(s, options, ssh_flags, 0);
+    ret = connect_to_ssh(s, options, ssh_flags, 0, errp);
     if (ret < 0) {
         goto err;
     }
@@ -655,7 +688,6 @@
                       Error **errp)
 {
     int r, ret;
-    Error *local_err = NULL;
     int64_t total_size = 0;
     QDict *uri_options = NULL;
     BDRVSSHState s;
@@ -674,17 +706,16 @@
     DPRINTF("total_size=%" PRIi64, total_size);
 
     uri_options = qdict_new();
-    r = parse_uri(filename, uri_options, &local_err);
+    r = parse_uri(filename, uri_options, errp);
     if (r < 0) {
-        qerror_report_err(local_err);
-        error_free(local_err);
         ret = r;
         goto out;
     }
 
     r = connect_to_ssh(&s, uri_options,
                        LIBSSH2_FXF_READ|LIBSSH2_FXF_WRITE|
-                       LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC, 0644);
+                       LIBSSH2_FXF_CREAT|LIBSSH2_FXF_TRUNC,
+                       0644, errp);
     if (r < 0) {
         ret = r;
         goto out;
@@ -694,7 +725,7 @@
         libssh2_sftp_seek64(s.sftp_handle, total_size-1);
         r2 = libssh2_sftp_write(s.sftp_handle, c, 1);
         if (r2 < 0) {
-            sftp_error_report(&s, "truncate failed");
+            sftp_error_setg(errp, &s, "truncate failed");
             ret = -EINVAL;
             goto out;
         }
diff --git a/block/stream.c b/block/stream.c
index dd0b4ac..91d18a2 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -60,7 +60,7 @@
     /* Must assign before bdrv_delete() to prevent traversing dangling pointer
      * while we delete backing image instances.
      */
-    top->backing_hd = base;
+    bdrv_set_backing_hd(top, base);
 
     while (intermediate) {
         BlockDriverState *unused;
@@ -72,7 +72,7 @@
 
         unused = intermediate;
         intermediate = intermediate->backing_hd;
-        unused->backing_hd = NULL;
+        bdrv_set_backing_hd(unused, NULL);
         bdrv_unref(unused);
     }
 
diff --git a/block/vhdx.c b/block/vhdx.c
index 509baaf..353c74d 100644
--- a/block/vhdx.c
+++ b/block/vhdx.c
@@ -473,7 +473,14 @@
         } else if (h2_seq > h1_seq) {
             s->curr_header = 1;
         } else {
-            goto fail;
+            /* The Microsoft Disk2VHD tool will create 2 identical
+             * headers, with identical sequence numbers.  If the headers are
+             * identical, don't consider the file corrupt */
+            if (!memcmp(header1, header2, sizeof(VHDXHeader))) {
+                s->curr_header = 0;
+            } else {
+                goto fail;
+            }
         }
     }
 
diff --git a/block/vmdk.c b/block/vmdk.c
index 480ea37..2b38f61 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -1534,7 +1534,7 @@
     int ret, i;
     BlockDriverState *bs = NULL;
     VMDK4Header header;
-    Error *local_err;
+    Error *local_err = NULL;
     uint32_t tmp, magic, grains, gd_sectors, gt_size, gt_count;
     uint32_t *gd_buf = NULL;
     int gd_buf_size;
@@ -1700,7 +1700,7 @@
 {
     int idx = 0;
     BlockDriverState *new_bs = NULL;
-    Error *local_err;
+    Error *local_err = NULL;
     char *desc = NULL;
     int64_t total_size = 0, filesize;
     const char *adapter_type = NULL;
@@ -1881,7 +1881,7 @@
     } else {
         ret = bdrv_create_file(filename, options, &local_err);
         if (ret < 0) {
-            error_setg_errno(errp, -ret, "Could not create image file");
+            error_propagate(errp, local_err);
             goto exit;
         }
     }
@@ -1889,7 +1889,7 @@
     ret = bdrv_open(&new_bs, filename, NULL, NULL,
                     BDRV_O_RDWR | BDRV_O_PROTOCOL, NULL, &local_err);
     if (ret < 0) {
-        error_setg_errno(errp, -ret, "Could not write description");
+        error_propagate(errp, local_err);
         goto exit;
     }
     ret = bdrv_pwrite(new_bs, desc_offset, desc, desc_len);
diff --git a/block/vvfat.c b/block/vvfat.c
index c3af7ff..3cda19f 100644
--- a/block/vvfat.c
+++ b/block/vvfat.c
@@ -787,7 +787,9 @@
 	    s->current_mapping->path=buffer;
 	    s->current_mapping->read_only =
 		(st.st_mode & (S_IWUSR | S_IWGRP | S_IWOTH)) == 0;
-	}
+        } else {
+            g_free(buffer);
+        }
     }
     closedir(dir);
 
@@ -831,7 +833,8 @@
 }
 
 static int init_directories(BDRVVVFATState* s,
-                            const char *dirname, int heads, int secs)
+                            const char *dirname, int heads, int secs,
+                            Error **errp)
 {
     bootsector_t* bootsector;
     mapping_t* mapping;
@@ -892,8 +895,8 @@
         if (mapping->mode & MODE_DIRECTORY) {
 	    mapping->begin = cluster;
 	    if(read_directory(s, i)) {
-		fprintf(stderr, "Could not read directory %s\n",
-			mapping->path);
+                error_setg(errp, "Could not read directory %s",
+                           mapping->path);
 		return -1;
 	    }
 	    mapping = array_get(&(s->mapping), i);
@@ -919,9 +922,10 @@
 	cluster = mapping->end;
 
 	if(cluster > s->cluster_count) {
-	    fprintf(stderr,"Directory does not fit in FAT%d (capacity %.2f MB)\n",
-		    s->fat_type, s->sector_count / 2000.0);
-	    return -EINVAL;
+            error_setg(errp,
+                       "Directory does not fit in FAT%d (capacity %.2f MB)",
+                       s->fat_type, s->sector_count / 2000.0);
+            return -1;
 	}
 
 	/* fix fat for entry */
@@ -979,7 +983,7 @@
 static BDRVVVFATState *vvv = NULL;
 #endif
 
-static int enable_write_target(BDRVVVFATState *s);
+static int enable_write_target(BDRVVVFATState *s, Error **errp);
 static int is_consistent(BDRVVVFATState *s);
 
 static void vvfat_rebind(BlockDriverState *bs)
@@ -1160,7 +1164,7 @@
     s->sector_count = cyls * heads * secs - (s->first_sectors_number - 1);
 
     if (qemu_opt_get_bool(opts, "rw", false)) {
-        ret = enable_write_target(s);
+        ret = enable_write_target(s, errp);
         if (ret < 0) {
             goto fail;
         }
@@ -1169,7 +1173,7 @@
 
     bs->total_sectors = cyls * heads * secs;
 
-    if (init_directories(s, dirname, heads, secs)) {
+    if (init_directories(s, dirname, heads, secs, errp)) {
         ret = -EIO;
         goto fail;
     }
@@ -1864,7 +1868,7 @@
 
 	if (s->used_clusters[cluster_num] & USED_ANY) {
 	    fprintf(stderr, "cluster %d used more than once\n", (int)cluster_num);
-	    return 0;
+            goto fail;
 	}
 	s->used_clusters[cluster_num] = USED_DIRECTORY;
 
@@ -2904,11 +2908,10 @@
     .bdrv_close         = write_target_close,
 };
 
-static int enable_write_target(BDRVVVFATState *s)
+static int enable_write_target(BDRVVVFATState *s, Error **errp)
 {
     BlockDriver *bdrv_qcow;
     QEMUOptionParameter *options;
-    Error *local_err = NULL;
     int ret;
     int size = sector2cluster(s, s->sector_count);
     s->used_clusters = calloc(size, 1);
@@ -2918,6 +2921,7 @@
     s->qcow_filename = g_malloc(1024);
     ret = get_tmp_filename(s->qcow_filename, 1024);
     if (ret < 0) {
+        error_setg_errno(errp, -ret, "can't create temporary file");
         goto err;
     }
 
@@ -2926,20 +2930,17 @@
     set_option_parameter_int(options, BLOCK_OPT_SIZE, s->sector_count * 512);
     set_option_parameter(options, BLOCK_OPT_BACKING_FILE, "fat:");
 
-    ret = bdrv_create(bdrv_qcow, s->qcow_filename, options, &local_err);
+    ret = bdrv_create(bdrv_qcow, s->qcow_filename, options, errp);
+    free_option_parameters(options);
     if (ret < 0) {
-        qerror_report_err(local_err);
-        error_free(local_err);
         goto err;
     }
 
     s->qcow = NULL;
     ret = bdrv_open(&s->qcow, s->qcow_filename, NULL, NULL,
-            BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH, bdrv_qcow,
-            &local_err);
+                    BDRV_O_RDWR | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH,
+                    bdrv_qcow, errp);
     if (ret < 0) {
-        qerror_report_err(local_err);
-        error_free(local_err);
         goto err;
     }
 
@@ -2947,7 +2948,7 @@
     unlink(s->qcow_filename);
 #endif
 
-    s->bs->backing_hd = bdrv_new("", &error_abort);
+    bdrv_set_backing_hd(s->bs, bdrv_new("", &error_abort));
     s->bs->backing_hd->drv = &vvfat_write_target;
     s->bs->backing_hd->opaque = g_malloc(sizeof(void*));
     *(void**)s->bs->backing_hd->opaque = s;
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
index 922cf56..b60b66d 100644
--- a/blockdev-nbd.c
+++ b/blockdev-nbd.c
@@ -27,8 +27,8 @@
     socklen_t addr_len = sizeof(addr);
 
     int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    if (fd >= 0) {
-        nbd_client_new(NULL, fd, nbd_client_put);
+    if (fd >= 0 && !nbd_client_new(NULL, fd, nbd_client_put)) {
+        close(fd);
     }
 }
 
diff --git a/blockdev.c b/blockdev.c
index 7810e9f..9b5261b 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -34,7 +34,6 @@
 #include "hw/block/block.h"
 #include "block/blockjob.h"
 #include "monitor/monitor.h"
-#include "qapi/qmp/qerror.h"
 #include "qemu/option.h"
 #include "qemu/config-file.h"
 #include "qapi/qmp/types.h"
@@ -288,6 +287,25 @@
     }
 }
 
+static inline int parse_enum_option(const char *lookup[], const char *buf,
+                                    int max, int def, Error **errp)
+{
+    int i;
+
+    if (!buf) {
+        return def;
+    }
+
+    for (i = 0; i < max; i++) {
+        if (!strcmp(buf, lookup[i])) {
+            return i;
+        }
+    }
+
+    error_setg(errp, "invalid parameter value: %s", buf);
+    return def;
+}
+
 static bool check_throttle_config(ThrottleConfig *cfg, Error **errp)
 {
     if (throttle_conflicting(cfg)) {
@@ -324,6 +342,7 @@
     QemuOpts *opts;
     const char *id;
     bool has_driver_specific_opts;
+    BlockdevDetectZeroesOptions detect_zeroes;
     BlockDriver *drv = NULL;
 
     /* Check common options by copying from bs_opts to opts, all other options
@@ -332,7 +351,7 @@
     opts = qemu_opts_create(&qemu_common_drive_opts, id, 1, &error);
     if (error) {
         error_propagate(errp, error);
-        return NULL;
+        goto err_no_opts;
     }
 
     qemu_opts_absorb_qdict(opts, bs_opts, &error);
@@ -452,6 +471,24 @@
         }
     }
 
+    detect_zeroes =
+        parse_enum_option(BlockdevDetectZeroesOptions_lookup,
+                          qemu_opt_get(opts, "detect-zeroes"),
+                          BLOCKDEV_DETECT_ZEROES_OPTIONS_MAX,
+                          BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF,
+                          &error);
+    if (error) {
+        error_propagate(errp, error);
+        goto early_err;
+    }
+
+    if (detect_zeroes == BLOCKDEV_DETECT_ZEROES_OPTIONS_UNMAP &&
+        !(bdrv_flags & BDRV_O_UNMAP)) {
+        error_setg(errp, "setting detect-zeroes to unmap is not allowed "
+                         "without setting discard operation to unmap");
+        goto early_err;
+    }
+
     /* init */
     dinfo = g_malloc0(sizeof(*dinfo));
     dinfo->id = g_strdup(qemu_opts_id(opts));
@@ -462,6 +499,7 @@
     }
     dinfo->bdrv->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
     dinfo->bdrv->read_only = ro;
+    dinfo->bdrv->detect_zeroes = detect_zeroes;
     dinfo->refcount = 1;
     if (serial != NULL) {
         dinfo->serial = g_strdup(serial);
@@ -526,8 +564,9 @@
     g_free(dinfo->id);
     g_free(dinfo);
 early_err:
-    QDECREF(bs_opts);
     qemu_opts_del(opts);
+err_no_opts:
+    QDECREF(bs_opts);
     return NULL;
 }
 
@@ -691,7 +730,7 @@
                                    &error_abort);
     qemu_opts_absorb_qdict(legacy_opts, bs_opts, &local_err);
     if (local_err) {
-        qerror_report_err(local_err);
+        error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
         goto fail;
     }
@@ -901,9 +940,10 @@
 
     /* Actual block device init: Functionality shared with blockdev-add */
     dinfo = blockdev_init(filename, bs_opts, &local_err);
+    bs_opts = NULL;
     if (dinfo == NULL) {
         if (local_err) {
-            qerror_report_err(local_err);
+            error_report("%s", error_get_pretty(local_err));
             error_free(local_err);
         }
         goto fail;
@@ -938,6 +978,7 @@
 
 fail:
     qemu_opts_del(legacy_opts);
+    QDECREF(bs_opts);
     return dinfo;
 }
 
@@ -1295,8 +1336,8 @@
         return;
     }
 
-    if (bdrv_in_use(state->old_bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, device);
+    if (bdrv_op_is_blocked(state->old_bs,
+                           BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT, errp)) {
         return;
     }
 
@@ -1518,8 +1559,7 @@
 
 static void eject_device(BlockDriverState *bs, int force, Error **errp)
 {
-    if (bdrv_in_use(bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_EJECT, errp)) {
         return;
     }
     if (!bdrv_dev_has_removable_media(bs)) {
@@ -1721,14 +1761,16 @@
 {
     const char *id = qdict_get_str(qdict, "id");
     BlockDriverState *bs;
+    Error *local_err = NULL;
 
     bs = bdrv_find(id);
     if (!bs) {
-        qerror_report(QERR_DEVICE_NOT_FOUND, id);
+        error_report("Device '%s' not found", id);
         return -1;
     }
-    if (bdrv_in_use(bs)) {
-        qerror_report(QERR_DEVICE_IN_USE, id);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_DRIVE_DEL, &local_err)) {
+        error_report("%s", error_get_pretty(local_err));
+        error_free(local_err);
         return -1;
     }
 
@@ -1849,6 +1891,10 @@
         return;
     }
 
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_STREAM, errp)) {
+        return;
+    }
+
     if (base) {
         base_bs = bdrv_find_backing_image(bs, base);
         if (base_bs == NULL) {
@@ -1893,6 +1939,10 @@
         return;
     }
 
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_COMMIT, errp)) {
+        return;
+    }
+
     /* default top_bs is the active layer */
     top_bs = bs;
 
@@ -1984,8 +2034,7 @@
         }
     }
 
-    if (bdrv_in_use(bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, device);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_BACKUP_SOURCE, errp)) {
         return;
     }
 
@@ -2118,8 +2167,7 @@
         }
     }
 
-    if (bdrv_in_use(bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, device);
+    if (bdrv_op_is_blocked(bs, BLOCK_OP_TYPE_MIRROR, errp)) {
         return;
     }
 
@@ -2455,6 +2503,10 @@
             .name = "copy-on-read",
             .type = QEMU_OPT_BOOL,
             .help = "copy read data from backing file into image file",
+        },{
+            .name = "detect-zeroes",
+            .type = QEMU_OPT_STRING,
+            .help = "try to optimize zero writes (off, on, unmap)",
         },
         { /* end of list */ }
     },
diff --git a/blockjob.c b/blockjob.c
index cd4784f..7d84ca1 100644
--- a/blockjob.c
+++ b/blockjob.c
@@ -41,14 +41,16 @@
 {
     BlockJob *job;
 
-    if (bs->job || bdrv_in_use(bs)) {
+    if (bs->job) {
         error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
         return NULL;
     }
     bdrv_ref(bs);
-    bdrv_set_in_use(bs, 1);
-
     job = g_malloc0(driver->instance_size);
+    error_setg(&job->blocker, "block device is in use by block job: %s",
+               BlockJobType_lookup[driver->job_type]);
+    bdrv_op_block_all(bs, job->blocker);
+
     job->driver        = driver;
     job->bs            = bs;
     job->cb            = cb;
@@ -63,8 +65,9 @@
         block_job_set_speed(job, speed, &local_err);
         if (local_err) {
             bs->job = NULL;
+            bdrv_op_unblock_all(bs, job->blocker);
+            error_free(job->blocker);
             g_free(job);
-            bdrv_set_in_use(bs, 0);
             error_propagate(errp, local_err);
             return NULL;
         }
@@ -79,8 +82,9 @@
     assert(bs->job == job);
     job->cb(job->opaque, ret);
     bs->job = NULL;
+    bdrv_op_unblock_all(bs, job->blocker);
+    error_free(job->blocker);
     g_free(job);
-    bdrv_set_in_use(bs, 0);
 }
 
 void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
diff --git a/bsd-user/main.c b/bsd-user/main.c
index 4ba61da..0e8c26c 100644
--- a/bsd-user/main.c
+++ b/bsd-user/main.c
@@ -1004,7 +1004,7 @@
 
 #if defined(TARGET_I386)
     env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK;
-    env->hflags |= HF_PE_MASK;
+    env->hflags |= HF_PE_MASK | HF_CPL_MASK;
     if (env->features[FEAT_1_EDX] & CPUID_SSE) {
         env->cr[4] |= CR4_OSFXSR_MASK;
         env->hflags |= HF_OSFXSR_MASK;
diff --git a/bsd-user/qemu.h b/bsd-user/qemu.h
index ddc74ed..9d90668 100644
--- a/bsd-user/qemu.h
+++ b/bsd-user/qemu.h
@@ -5,6 +5,7 @@
 #include <string.h>
 
 #include "cpu.h"
+#include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
diff --git a/configure b/configure
index 605a0ec..0e516f9 100755
--- a/configure
+++ b/configure
@@ -2,26 +2,28 @@
 #
 # qemu configure script (c) 2003 Fabrice Bellard
 #
-# set temporary file name
-if test ! -z "$TMPDIR" ; then
-    TMPDIR1="${TMPDIR}"
-elif test ! -z "$TEMPDIR" ; then
-    TMPDIR1="${TEMPDIR}"
-else
-    TMPDIR1="/tmp"
+
+# Temporary directory used for files created while
+# configure runs. Since it is in the build directory
+# we can safely blow away any previous version of it
+# (and we need not jump through hoops to try to delete
+# it when configure exits.)
+TMPDIR1="config-temp"
+rm -rf "${TMPDIR1}"
+mkdir -p "${TMPDIR1}"
+if [ $? -ne 0 ]; then
+    echo "ERROR: failed to create temporary directory"
+    exit 1
 fi
 
-TMPC="${TMPDIR1}/qemu-conf-${RANDOM}-$$-${RANDOM}.c"
-TMPB="qemu-conf-${RANDOM}-$$-${RANDOM}"
+TMPB="qemu-conf"
+TMPC="${TMPDIR1}/${TMPB}.c"
 TMPO="${TMPDIR1}/${TMPB}.o"
 TMPCXX="${TMPDIR1}/${TMPB}.cxx"
 TMPL="${TMPDIR1}/${TMPB}.lo"
 TMPA="${TMPDIR1}/lib${TMPB}.la"
-TMPE="${TMPDIR1}/qemu-conf-${RANDOM}-$$-${RANDOM}.exe"
+TMPE="${TMPDIR1}/${TMPB}.exe"
 
-# NB: do not call "exit" in the trap handler; this is buggy with some shells;
-# see <1285349658-3122-1-git-send-email-loic.minier@linaro.org>
-trap "rm -f $TMPC $TMPO $TMPCXX $TMPE" EXIT INT QUIT TERM
 rm -f config.log
 
 # Print a helpful header at the top of config.log
@@ -317,7 +319,7 @@
 glusterfs_zerofill="no"
 virtio_blk_data_plane=""
 gtk=""
-gtkabi="2.0"
+gtkabi=""
 vte=""
 tpm="no"
 libssh2=""
@@ -1970,6 +1972,18 @@
 ##########################################
 # GTK probe
 
+if test "$gtkabi" = ""; then
+    # The GTK ABI was not specified explicitly, so try whether 2.0 is available.
+    # Use 3.0 as a fallback if that is available.
+    if $pkg_config --exists "gtk+-2.0 >= 2.18.0"; then
+        gtkabi=2.0
+    elif $pkg_config --exists "gtk+-3.0 >= 3.0.0"; then
+        gtkabi=3.0
+    else
+        gtkabi=2.0
+    fi
+fi
+
 if test "$gtk" != "no"; then
     gtkpackage="gtk+-$gtkabi"
     if test "$gtkabi" = "3.0" ; then
@@ -1983,7 +1997,7 @@
         libs_softmmu="$gtk_libs $libs_softmmu"
         gtk="yes"
     elif test "$gtk" = "yes"; then
-        feature_not_found "gtk" "Install gtk2 or gtk3 (requires --with-gtkabi=3.0 option to configure) devel"
+        feature_not_found "gtk" "Install gtk2 or gtk3 devel"
     else
         gtk="no"
     fi
@@ -2006,7 +2020,11 @@
         libs_softmmu="$vte_libs $libs_softmmu"
         vte="yes"
     elif test "$vte" = "yes"; then
-        feature_not_found "vte" "Install libvte or libvte-2.90 (requires --with-gtkabi=3.0 option to configure) devel"
+        if test "$gtkabi" = "3.0"; then
+            feature_not_found "vte" "Install libvte-2.90 devel"
+        else
+            feature_not_found "vte" "Install libvte devel"
+        fi
     else
         vte="no"
     fi
@@ -4029,11 +4047,14 @@
 if test "$pie" = "no" ; then
   textseg_addr=
   case "$cpu" in
-    arm | hppa | i386 | m68k | ppc | ppc64 | s390* | sparc | sparc64 | x86_64 | x32)
+    arm | i386 | ppc* | s390* | sparc* | x86_64 | x32)
+      # ??? Rationale for choosing this address
       textseg_addr=0x60000000
       ;;
     mips)
-      textseg_addr=0x400000
+      # A 256M aligned address, high in the address space, with enough
+      # room for the code_gen_buffer above it before the stack.
+      textseg_addr=0x60000000
       ;;
   esac
   if [ -n "$textseg_addr" ]; then
@@ -5219,3 +5240,4 @@
 echo >>config.status
 chmod +x config.status
 
+rm -r "$TMPDIR1"
diff --git a/cputlb.c b/cputlb.c
index 7bd3573..afd3705 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -22,11 +22,13 @@
 #include "exec/exec-all.h"
 #include "exec/memory.h"
 #include "exec/address-spaces.h"
+#include "exec/cpu_ldst.h"
 
 #include "exec/cputlb.h"
 
 #include "exec/memory-internal.h"
 #include "exec/ram_addr.h"
+#include "tcg/tcg.h"
 
 //#define DEBUG_TLB
 //#define DEBUG_TLB_CHECK
@@ -330,21 +332,36 @@
     return qemu_ram_addr_from_host_nofail(p);
 }
 
+#define MMUSUFFIX _mmu
+
+#define SHIFT 0
+#include "softmmu_template.h"
+
+#define SHIFT 1
+#include "softmmu_template.h"
+
+#define SHIFT 2
+#include "softmmu_template.h"
+
+#define SHIFT 3
+#include "softmmu_template.h"
+#undef MMUSUFFIX
+
 #define MMUSUFFIX _cmmu
-#undef GETPC
-#define GETPC() ((uintptr_t)0)
+#undef GETPC_ADJ
+#define GETPC_ADJ 0
+#undef GETRA
+#define GETRA() ((uintptr_t)0)
 #define SOFTMMU_CODE_ACCESS
 
 #define SHIFT 0
-#include "exec/softmmu_template.h"
+#include "softmmu_template.h"
 
 #define SHIFT 1
-#include "exec/softmmu_template.h"
+#include "softmmu_template.h"
 
 #define SHIFT 2
-#include "exec/softmmu_template.h"
+#include "softmmu_template.h"
 
 #define SHIFT 3
-#include "exec/softmmu_template.h"
-
-#undef env
+#include "softmmu_template.h"
diff --git a/default-configs/s390x-softmmu.mak b/default-configs/s390x-softmmu.mak
index d843dc0..126d88d 100644
--- a/default-configs/s390x-softmmu.mak
+++ b/default-configs/s390x-softmmu.mak
@@ -1,3 +1,4 @@
 CONFIG_VIRTIO=y
 CONFIG_SCLPCONSOLE=y
-CONFIG_S390_FLIC=$(CONFIG_KVM)
+CONFIG_S390_FLIC=y
+CONFIG_S390_FLIC_KVM=$(CONFIG_KVM)
diff --git a/dma-helpers.c b/dma-helpers.c
index 5f421e9..53cbe92 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -143,12 +143,12 @@
 
     dbs->acb = NULL;
     dbs->sector_num += dbs->iov.size / 512;
-    dma_bdrv_unmap(dbs);
 
     if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
         dma_complete(dbs, ret);
         return;
     }
+    dma_bdrv_unmap(dbs);
 
     while (dbs->sg_cur_index < dbs->sg->nsg) {
         cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
diff --git a/docs/multiseat.txt b/docs/multiseat.txt
new file mode 100644
index 0000000..67151e0
--- /dev/null
+++ b/docs/multiseat.txt
@@ -0,0 +1,102 @@
+
+multiseat howto (with some multihead coverage)
+==============================================
+
+host side
+---------
+
+First you must compile qemu with a user interface supporting
+multihead/multiseat and input event routing.  Right now this
+list includes sdl2 and gtk (both 2+3):
+
+  ./configure --enable-sdl --with-sdlabi=2.0
+
+or
+
+  ./configure --enable-gtk
+
+
+Next put together the qemu command line:
+
+qemu	-enable-kvm -usb $memory $disk $whatever \
+	-display [ sdl | gtk ] \
+	-vga std \
+	-device usb-tablet
+
+That is it for the first head, which will use the standard vga, the
+standard ps/2 keyboard (implicitly there) and the usb-tablet.  Now the
+additional switches for the second head:
+
+	-device pci-bridge,addr=12.0,chassis_nr=2,id=head.2 \
+	-device secondary-vga,bus=head.2,addr=02.0,id=video.2 \
+	-device nec-usb-xhci,bus=head.2,addr=0f.0,id=usb.2 \
+	-device usb-kbd,bus=usb.2.0,port=1,display=video.2 \
+	-device usb-tablet,bus=usb.2.0,port=2,display=video.2
+
+This places a pci bridge in slot 12, connects a display adapter and
+xhci (usb) controller to the bridge.  Then it adds a usb keyboard and
+usb mouse, both connected to the xhci and linked to the display.
+
+The "display=video2" sets up the input routing.  Any input coming from
+the window which belongs to the video.2 display adapter will be routed
+to these input devices.
+
+The sdl2 ui will start up with two windows, one for each display
+device.  The gtk ui will start with a single window and each display
+in a separate tab.  You can either simply switch tabs to switch heads,
+or use the "View / Detach tab" menu item to move one of the displays
+to its own window so you can see both display devices side-by-side.
+
+Note on spice: Spice handles multihead just fine.  But it can't do
+multiseat.  For tablet events the event source is sent to the spice
+agent.  But qemu can't figure it, so it can't do input routing.
+Fixing this needs a new or extended input interface between
+libspice-server and qemu.  For keyboard events it is even worse:  The
+event source isn't included in the spice protocol, so the wire
+protocol must be extended to support this.
+
+
+guest side
+----------
+
+You need a pretty recent linux guest.  systemd with loginctl.  kernel
+3.14+ with CONFIG_DRM_BOCHS enabled.  Fedora 20 will do.  Must be
+fully updated for the new kernel though, i.e. the live iso doesn't cut
+it.
+
+Now we'll have to configure the guest.  Boot and login.  "lspci -vt"
+should list the pci bridge with the display adapter and usb controller:
+
+    [root@fedora ~]# lspci -vt
+    -[0000:00]-+-00.0  Intel Corporation 440FX - 82441FX PMC [Natoma]
+               [ ... ]
+               \-12.0-[01]--+-02.0  Device 1234:1111
+                            \-0f.0  NEC Corporation USB 3.0 Host Controller
+
+Good.  Now lets tell the system that the pci bridge and all devices
+below it belong to a separate seat by dropping a file into
+/etc/udev/rules.d:
+
+    [root@fedora ~]# cat /etc/udev/rules.d/70-qemu-autoseat.rules
+    SUBSYSTEMS=="pci", DEVPATH=="*/0000:00:12.0", TAG+="seat", ENV{ID_AUTOSEAT}="1"
+
+Reboot.  System should come up with two seats.  With loginctl you can
+check the configuration:
+
+    [root@fedora ~]# loginctl list-seats
+    SEAT
+    seat0
+    seat-pci-pci-0000_00_12_0
+
+    2 seats listed.
+
+You can use "loginctl seat-status seat-pci-pci-0000_00_12_0" to list
+the devices attached to the seat.
+
+Background info is here:
+  http://www.freedesktop.org/wiki/Software/systemd/multiseat/
+
+Enjoy!
+
+--
+Gerd Hoffmann <kraxel@redhat.com>
diff --git a/docs/specs/qcow2.txt b/docs/specs/qcow2.txt
index f19536a..3f713a6 100644
--- a/docs/specs/qcow2.txt
+++ b/docs/specs/qcow2.txt
@@ -107,8 +107,9 @@
 
          96 -  99:  refcount_order
                     Describes the width of a reference count block entry (width
-                    in bits = 1 << refcount_order). For version 2 images, the
-                    order is always assumed to be 4 (i.e. the width is 16 bits).
+                    in bits: refcount_bits = 1 << refcount_order). For version 2
+                    images, the order is always assumed to be 4
+                    (i.e. refcount_bits = 16).
 
         100 - 103:  header_length
                     Length of the header structure in bytes. For version 2
diff --git a/hmp.c b/hmp.c
index a9d0236..ccc35d4 100644
--- a/hmp.c
+++ b/hmp.c
@@ -341,6 +341,11 @@
                            info->value->inserted->backing_file_depth);
         }
 
+        if (info->value->inserted->detect_zeroes != BLOCKDEV_DETECT_ZEROES_OPTIONS_OFF) {
+            monitor_printf(mon, "    Detect zeroes:    %s\n",
+                           BlockdevDetectZeroesOptions_lookup[info->value->inserted->detect_zeroes]);
+        }
+
         if (info->value->inserted->bps
             || info->value->inserted->bps_rd
             || info->value->inserted->bps_wr
diff --git a/hw/9pfs/virtio-9p-device.c b/hw/9pfs/virtio-9p-device.c
index 15a4983..653762a 100644
--- a/hw/9pfs/virtio-9p-device.c
+++ b/hw/9pfs/virtio-9p-device.c
@@ -34,7 +34,7 @@
 
     len = strlen(s->tag);
     cfg = g_malloc0(sizeof(struct virtio_9p_config) + len);
-    stw_raw(&cfg->tag_len, len);
+    stw_p(&cfg->tag_len, len);
     /* We don't copy the terminating null to config space */
     memcpy(cfg->tag, s->tag, len);
     memcpy(config, cfg, s->config_size);
diff --git a/hw/alpha/dp264.c b/hw/alpha/dp264.c
index 1351ba5..b178a03 100644
--- a/hw/alpha/dp264.c
+++ b/hw/alpha/dp264.c
@@ -43,13 +43,13 @@
     return (slot + 1) * 4 + irq_num;
 }
 
-static void clipper_init(QEMUMachineInitArgs *args)
+static void clipper_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     AlphaCPU *cpus[4];
     PCIBus *pci_bus;
     ISABus *isa_bus;
diff --git a/hw/arm/collie.c b/hw/arm/collie.c
index 8878b0e..ed7851f 100644
--- a/hw/arm/collie.c
+++ b/hw/arm/collie.c
@@ -23,12 +23,12 @@
     .ram_size = 0x20000000,
 };
 
-static void collie_init(QEMUMachineInitArgs *args)
+static void collie_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     StrongARMState *s;
     DriveInfo *dinfo;
     MemoryRegion *sysmem = get_system_memory();
diff --git a/hw/arm/cubieboard.c b/hw/arm/cubieboard.c
index 9d158c7..e2260e3 100644
--- a/hw/arm/cubieboard.c
+++ b/hw/arm/cubieboard.c
@@ -30,7 +30,7 @@
     MemoryRegion sdram;
 } CubieBoardState;
 
-static void cubieboard_init(QEMUMachineInitArgs *args)
+static void cubieboard_init(MachineState *machine)
 {
     CubieBoardState *s = g_new(CubieBoardState, 1);
     Error *err = NULL;
@@ -63,14 +63,15 @@
         exit(1);
     }
 
-    memory_region_init_ram(&s->sdram, NULL, "cubieboard.ram", args->ram_size);
+    memory_region_init_ram(&s->sdram, NULL, "cubieboard.ram",
+                           machine->ram_size);
     vmstate_register_ram_global(&s->sdram);
     memory_region_add_subregion(get_system_memory(), AW_A10_SDRAM_BASE,
                                 &s->sdram);
 
-    cubieboard_binfo.ram_size = args->ram_size;
-    cubieboard_binfo.kernel_filename = args->kernel_filename;
-    cubieboard_binfo.kernel_cmdline = args->kernel_cmdline;
+    cubieboard_binfo.ram_size = machine->ram_size;
+    cubieboard_binfo.kernel_filename = machine->kernel_filename;
+    cubieboard_binfo.kernel_cmdline = machine->kernel_cmdline;
     arm_load_kernel(&s->a10->cpu, &cubieboard_binfo);
 }
 
diff --git a/hw/arm/digic_boards.c b/hw/arm/digic_boards.c
index 32fc30a..d1424ee 100644
--- a/hw/arm/digic_boards.c
+++ b/hw/arm/digic_boards.c
@@ -143,7 +143,7 @@
     .rom1_def_filename = "canon-a1100-rom1.bin",
 };
 
-static void canon_a1100_init(QEMUMachineInitArgs *args)
+static void canon_a1100_init(MachineState *machine)
 {
     digic4_board_init(&digic4_board_canon_a1100);
 }
diff --git a/hw/arm/exynos4_boards.c b/hw/arm/exynos4_boards.c
index 26cedec..d644db1 100644
--- a/hw/arm/exynos4_boards.c
+++ b/hw/arm/exynos4_boards.c
@@ -94,7 +94,7 @@
     }
 }
 
-static Exynos4210State *exynos4_boards_init_common(QEMUMachineInitArgs *args,
+static Exynos4210State *exynos4_boards_init_common(MachineState *machine,
                                                    Exynos4BoardType board_type)
 {
     if (smp_cpus != EXYNOS4210_NCPUS && !qtest_enabled()) {
@@ -108,9 +108,9 @@
     exynos4_board_binfo.board_id = exynos4_board_id[board_type];
     exynos4_board_binfo.smp_bootreg_addr =
             exynos4_board_smp_bootreg_addr[board_type];
-    exynos4_board_binfo.kernel_filename = args->kernel_filename;
-    exynos4_board_binfo.initrd_filename = args->initrd_filename;
-    exynos4_board_binfo.kernel_cmdline = args->kernel_cmdline;
+    exynos4_board_binfo.kernel_filename = machine->kernel_filename;
+    exynos4_board_binfo.initrd_filename = machine->initrd_filename;
+    exynos4_board_binfo.kernel_cmdline = machine->kernel_cmdline;
     exynos4_board_binfo.gic_cpu_if_addr =
             EXYNOS4210_SMP_PRIVATE_BASE_ADDR + 0x100;
 
@@ -120,24 +120,24 @@
             " initrd_filename: %s\n",
             exynos4_board_ram_size[board_type] / 1048576,
             exynos4_board_ram_size[board_type],
-            args->kernel_filename,
-            args->kernel_cmdline,
-            args->initrd_filename);
+            machine->kernel_filename,
+            machine->kernel_cmdline,
+            machine->initrd_filename);
 
     return exynos4210_init(get_system_memory(),
             exynos4_board_ram_size[board_type]);
 }
 
-static void nuri_init(QEMUMachineInitArgs *args)
+static void nuri_init(MachineState *machine)
 {
-    exynos4_boards_init_common(args, EXYNOS4_BOARD_NURI);
+    exynos4_boards_init_common(machine, EXYNOS4_BOARD_NURI);
 
     arm_load_kernel(ARM_CPU(first_cpu), &exynos4_board_binfo);
 }
 
-static void smdkc210_init(QEMUMachineInitArgs *args)
+static void smdkc210_init(MachineState *machine)
 {
-    Exynos4210State *s = exynos4_boards_init_common(args,
+    Exynos4210State *s = exynos4_boards_init_common(machine,
                                                     EXYNOS4_BOARD_SMDKC210);
 
     lan9215_init(SMDK_LAN9118_BASE_ADDR,
diff --git a/hw/arm/gumstix.c b/hw/arm/gumstix.c
index aeea172..3f8465e 100644
--- a/hw/arm/gumstix.c
+++ b/hw/arm/gumstix.c
@@ -46,7 +46,7 @@
 
 static const int sector_len = 128 * 1024;
 
-static void connex_init(QEMUMachineInitArgs *args)
+static void connex_init(MachineState *machine)
 {
     PXA2xxState *cpu;
     DriveInfo *dinfo;
@@ -83,9 +83,9 @@
                     qdev_get_gpio_in(cpu->gpio, 36));
 }
 
-static void verdex_init(QEMUMachineInitArgs *args)
+static void verdex_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
     PXA2xxState *cpu;
     DriveInfo *dinfo;
     int be;
diff --git a/hw/arm/highbank.c b/hw/arm/highbank.c
index 24231e5..8340434 100644
--- a/hw/arm/highbank.c
+++ b/hw/arm/highbank.c
@@ -199,13 +199,13 @@
  * 32-bit host, set the reg value of memory to 0xf7ff00000 in the
  * device tree and pass -m 2047 to QEMU.
  */
-static void calxeda_init(QEMUMachineInitArgs *args, enum cxmachines machine)
+static void calxeda_init(MachineState *machine, enum cxmachines machine_id)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     DeviceState *dev = NULL;
     SysBusDevice *busdev;
     qemu_irq pic[128];
@@ -217,7 +217,7 @@
     char *sysboot_filename;
 
     if (!cpu_model) {
-        switch (machine) {
+        switch (machine_id) {
         case CALXEDA_HIGHBANK:
             cpu_model = "cortex-a9";
             break;
@@ -274,7 +274,7 @@
         }
     }
 
-    switch (machine) {
+    switch (machine_id) {
     case CALXEDA_HIGHBANK:
         dev = qdev_create(NULL, "l2x0");
         qdev_init_nofail(dev);
@@ -359,14 +359,14 @@
     arm_load_kernel(ARM_CPU(first_cpu), &highbank_binfo);
 }
 
-static void highbank_init(QEMUMachineInitArgs *args)
+static void highbank_init(MachineState *machine)
 {
-    calxeda_init(args, CALXEDA_HIGHBANK);
+    calxeda_init(machine, CALXEDA_HIGHBANK);
 }
 
-static void midway_init(QEMUMachineInitArgs *args)
+static void midway_init(MachineState *machine)
 {
-    calxeda_init(args, CALXEDA_MIDWAY);
+    calxeda_init(machine, CALXEDA_MIDWAY);
 }
 
 static QEMUMachine highbank_machine = {
diff --git a/hw/arm/integratorcp.c b/hw/arm/integratorcp.c
index 912af96..0e476c3 100644
--- a/hw/arm/integratorcp.c
+++ b/hw/arm/integratorcp.c
@@ -461,13 +461,13 @@
     .board_id = 0x113,
 };
 
-static void integratorcp_init(QEMUMachineInitArgs *args)
+static void integratorcp_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/arm/kzm.c b/hw/arm/kzm.c
index 99d33cb..0555d12 100644
--- a/hw/arm/kzm.c
+++ b/hw/arm/kzm.c
@@ -70,13 +70,13 @@
     .board_id = 1722,
 };
 
-static void kzm_init(QEMUMachineInitArgs *args)
+static void kzm_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/arm/mainstone.c b/hw/arm/mainstone.c
index d8e075e..44f1873 100644
--- a/hw/arm/mainstone.c
+++ b/hw/arm/mainstone.c
@@ -105,7 +105,7 @@
 };
 
 static void mainstone_common_init(MemoryRegion *address_space_mem,
-                                  QEMUMachineInitArgs *args,
+                                  MachineState *machine,
                                   enum mainstone_model_e model, int arm_id)
 {
     uint32_t sector_len = 256 * 1024;
@@ -116,7 +116,7 @@
     int i;
     int be;
     MemoryRegion *rom = g_new(MemoryRegion, 1);
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
 
     if (!cpu_model)
         cpu_model = "pxa270-c5";
@@ -175,16 +175,16 @@
     smc91c111_init(&nd_table[0], MST_ETH_PHYS,
                     qdev_get_gpio_in(mst_irq, ETHERNET_IRQ));
 
-    mainstone_binfo.kernel_filename = args->kernel_filename;
-    mainstone_binfo.kernel_cmdline = args->kernel_cmdline;
-    mainstone_binfo.initrd_filename = args->initrd_filename;
+    mainstone_binfo.kernel_filename = machine->kernel_filename;
+    mainstone_binfo.kernel_cmdline = machine->kernel_cmdline;
+    mainstone_binfo.initrd_filename = machine->initrd_filename;
     mainstone_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &mainstone_binfo);
 }
 
-static void mainstone_init(QEMUMachineInitArgs *args)
+static void mainstone_init(MachineState *machine)
 {
-    mainstone_common_init(get_system_memory(), args, mainstone, 0x196);
+    mainstone_common_init(get_system_memory(), machine, mainstone, 0x196);
 }
 
 static QEMUMachine mainstone2_machine = {
diff --git a/hw/arm/musicpal.c b/hw/arm/musicpal.c
index 2a27a19d..6a134f2 100644
--- a/hw/arm/musicpal.c
+++ b/hw/arm/musicpal.c
@@ -1569,12 +1569,12 @@
     .board_id = 0x20e,
 };
 
-static void musicpal_init(QEMUMachineInitArgs *args)
+static void musicpal_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     ARMCPU *cpu;
     qemu_irq pic[32];
     DeviceState *dev;
diff --git a/hw/arm/nseries.c b/hw/arm/nseries.c
index c28f895..4f092d6 100644
--- a/hw/arm/nseries.c
+++ b/hw/arm/nseries.c
@@ -239,8 +239,9 @@
     int code = s->keymap[keycode & 0x7f];
 
     if (code == -1) {
-        if ((keycode & 0x7f) == RETU_KEYCODE)
+        if ((keycode & 0x7f) == RETU_KEYCODE) {
             retu_key_event(s->retu, !(keycode & 0x80));
+        }
         return;
     }
 
@@ -280,11 +281,14 @@
     s->ts.opaque = s->ts.chip->opaque;
     s->ts.txrx = tsc210x_txrx;
 
-    for (i = 0; i < 0x80; i ++)
+    for (i = 0; i < 0x80; i++) {
         s->keymap[i] = -1;
-    for (i = 0; i < 0x10; i ++)
-        if (n800_keys[i] >= 0)
+    }
+    for (i = 0; i < 0x10; i++) {
+        if (n800_keys[i] >= 0) {
             s->keymap[n800_keys[i]] = i;
+        }
+    }
 
     qemu_add_kbd_event_handler(n800_key_event, s);
 
@@ -308,8 +312,9 @@
     int code = s->keymap[keycode & 0x7f];
 
     if (code == -1) {
-        if ((keycode & 0x7f) == RETU_KEYCODE)
+        if ((keycode & 0x7f) == RETU_KEYCODE) {
             retu_key_event(s->retu, !(keycode & 0x80));
+        }
         return;
     }
 
@@ -388,11 +393,14 @@
     qemu_irq kbd_irq = qdev_get_gpio_in(s->mpu->gpio, N810_KEYBOARD_GPIO);
     int i;
 
-    for (i = 0; i < 0x80; i ++)
+    for (i = 0; i < 0x80; i++) {
         s->keymap[i] = -1;
-    for (i = 0; i < 0x80; i ++)
-        if (n810_keys[i] > 0)
+    }
+    for (i = 0; i < 0x80; i++) {
+        if (n810_keys[i] > 0) {
             s->keymap[n810_keys[i]] = i;
+        }
+    }
 
     qemu_add_kbd_event_handler(n810_key_event, s);
 
@@ -449,17 +457,20 @@
     struct mipid_s *s = (struct mipid_s *) opaque;
     uint8_t ret;
 
-    if (len > 9)
+    if (len > 9) {
         hw_error("%s: FIXME: bad SPI word width %i\n", __FUNCTION__, len);
+    }
 
-    if (s->p >= ARRAY_SIZE(s->resp))
+    if (s->p >= ARRAY_SIZE(s->resp)) {
         ret = 0;
-    else
-        ret = s->resp[s->p ++];
-    if (s->pm --> 0)
+    } else {
+        ret = s->resp[s->p++];
+    }
+    if (s->pm-- > 0) {
         s->param[s->pm] = cmd;
-    else
+    } else {
         s->cmd = cmd;
+    }
 
     switch (s->cmd) {
     case 0x00:	/* NOP */
@@ -560,15 +571,17 @@
         goto bad_cmd;
 
     case 0x25:	/* WRCNTR */
-        if (s->pm < 0)
+        if (s->pm < 0) {
             s->pm = 1;
+        }
         goto bad_cmd;
 
     case 0x26:	/* GAMSET */
-        if (!s->pm)
+        if (!s->pm) {
             s->gamma = ffs(s->param[0] & 0xf) - 1;
-        else if (s->pm < 0)
+        } else if (s->pm < 0) {
             s->pm = 1;
+        }
         break;
 
     case 0x28:	/* DISPOFF */
@@ -591,10 +604,11 @@
         s->te = 0;
         break;
     case 0x35:	/* TEON */
-        if (!s->pm)
+        if (!s->pm) {
             s->te = 1;
-        else if (s->pm < 0)
+        } else if (s->pm < 0) {
             s->pm = 1;
+        }
         break;
 
     case 0x36:	/* MADCTR */
@@ -613,8 +627,9 @@
 
     case 0xb0:	/* CLKINT / DISCTL */
     case 0xb1:	/* CLKEXT */
-        if (s->pm < 0)
+        if (s->pm < 0) {
             s->pm = 2;
+        }
         break;
 
     case 0xb4:	/* FRMSEL */
@@ -635,8 +650,9 @@
         break;
 
     case 0xc2:	/* IFMOD */
-        if (s->pm < 0)
+        if (s->pm < 0) {
             s->pm = 2;
+        }
         break;
 
     case 0xc6:	/* PWRCTL */
@@ -834,118 +850,119 @@
 
     strcpy((void *) (p + 8), "F5");
 
-    stl_raw(p + 10, 0x04f70000);
+    stl_p(p + 10, 0x04f70000);
     strcpy((void *) (p + 9), "RX-34");
 
     /* RAM size in MB? */
-    stl_raw(p + 12, 0x80);
+    stl_p(p + 12, 0x80);
 
     /* Pointer to the list of tags */
-    stl_raw(p + 13, OMAP2_SRAM_BASE + 0x9000);
+    stl_p(p + 13, OMAP2_SRAM_BASE + 0x9000);
 
     /* The NOLO tags start here */
     p = sram_base + 0x9000;
 #define ADD_TAG(tag, len)				\
-    stw_raw((uint16_t *) p + 0, tag);			\
-    stw_raw((uint16_t *) p + 1, len); p ++;		\
-    stl_raw(p ++, OMAP2_SRAM_BASE | (((void *) v - sram_base) & 0xffff));
+    stw_p((uint16_t *) p + 0, tag);			\
+    stw_p((uint16_t *) p + 1, len); p++;		\
+    stl_p(p++, OMAP2_SRAM_BASE | (((void *) v - sram_base) & 0xffff));
 
     /* OMAP STI console? Pin out settings? */
     ADD_TAG(0x6e01, 414);
-    for (i = 0; i < ARRAY_SIZE(n800_pinout); i ++)
-        stl_raw(v ++, n800_pinout[i]);
+    for (i = 0; i < ARRAY_SIZE(n800_pinout); i++) {
+        stl_p(v++, n800_pinout[i]);
+    }
 
     /* Kernel memsize? */
     ADD_TAG(0x6e05, 1);
-    stl_raw(v ++, 2);
+    stl_p(v++, 2);
 
     /* NOLO serial console */
     ADD_TAG(0x6e02, 4);
-    stl_raw(v ++, XLDR_LL_UART);	/* UART number (1 - 3) */
+    stl_p(v++, XLDR_LL_UART);		/* UART number (1 - 3) */
 
 #if 0
     /* CBUS settings (Retu/AVilma) */
     ADD_TAG(0x6e03, 6);
-    stw_raw((uint16_t *) v + 0, 65);	/* CBUS GPIO0 */
-    stw_raw((uint16_t *) v + 1, 66);	/* CBUS GPIO1 */
-    stw_raw((uint16_t *) v + 2, 64);	/* CBUS GPIO2 */
+    stw_p((uint16_t *) v + 0, 65);	/* CBUS GPIO0 */
+    stw_p((uint16_t *) v + 1, 66);	/* CBUS GPIO1 */
+    stw_p((uint16_t *) v + 2, 64);	/* CBUS GPIO2 */
     v += 2;
 #endif
 
     /* Nokia ASIC BB5 (Retu/Tahvo) */
     ADD_TAG(0x6e0a, 4);
-    stw_raw((uint16_t *) v + 0, 111);	/* "Retu" interrupt GPIO */
-    stw_raw((uint16_t *) v + 1, 108);	/* "Tahvo" interrupt GPIO */
-    v ++;
+    stw_p((uint16_t *) v + 0, 111);	/* "Retu" interrupt GPIO */
+    stw_p((uint16_t *) v + 1, 108);	/* "Tahvo" interrupt GPIO */
+    v++;
 
     /* LCD console? */
     ADD_TAG(0x6e04, 4);
-    stw_raw((uint16_t *) v + 0, 30);	/* ??? */
-    stw_raw((uint16_t *) v + 1, 24);	/* ??? */
-    v ++;
+    stw_p((uint16_t *) v + 0, 30);	/* ??? */
+    stw_p((uint16_t *) v + 1, 24);	/* ??? */
+    v++;
 
 #if 0
     /* LCD settings */
     ADD_TAG(0x6e06, 2);
-    stw_raw((uint16_t *) (v ++), 15);	/* ??? */
+    stw_p((uint16_t *) (v++), 15);	/* ??? */
 #endif
 
     /* I^2C (Menelaus) */
     ADD_TAG(0x6e07, 4);
-    stl_raw(v ++, 0x00720000);		/* ??? */
+    stl_p(v++, 0x00720000);		/* ??? */
 
     /* Unknown */
     ADD_TAG(0x6e0b, 6);
-    stw_raw((uint16_t *) v + 0, 94);	/* ??? */
-    stw_raw((uint16_t *) v + 1, 23);	/* ??? */
-    stw_raw((uint16_t *) v + 2, 0);	/* ??? */
+    stw_p((uint16_t *) v + 0, 94);	/* ??? */
+    stw_p((uint16_t *) v + 1, 23);	/* ??? */
+    stw_p((uint16_t *) v + 2, 0);	/* ??? */
     v += 2;
 
     /* OMAP gpio switch info */
     ADD_TAG(0x6e0c, 80);
     strcpy((void *) v, "bat_cover");	v += 3;
-    stw_raw((uint16_t *) v + 0, 110);	/* GPIO num ??? */
-    stw_raw((uint16_t *) v + 1, 1);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 0, 110);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 1, 1);	/* GPIO num ??? */
     v += 2;
     strcpy((void *) v, "cam_act");	v += 3;
-    stw_raw((uint16_t *) v + 0, 95);	/* GPIO num ??? */
-    stw_raw((uint16_t *) v + 1, 32);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 0, 95);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 1, 32);	/* GPIO num ??? */
     v += 2;
     strcpy((void *) v, "cam_turn");	v += 3;
-    stw_raw((uint16_t *) v + 0, 12);	/* GPIO num ??? */
-    stw_raw((uint16_t *) v + 1, 33);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 0, 12);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 1, 33);	/* GPIO num ??? */
     v += 2;
     strcpy((void *) v, "headphone");	v += 3;
-    stw_raw((uint16_t *) v + 0, 107);	/* GPIO num ??? */
-    stw_raw((uint16_t *) v + 1, 17);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 0, 107);	/* GPIO num ??? */
+    stw_p((uint16_t *) v + 1, 17);	/* GPIO num ??? */
     v += 2;
 
     /* Bluetooth */
     ADD_TAG(0x6e0e, 12);
-    stl_raw(v ++, 0x5c623d01);		/* ??? */
-    stl_raw(v ++, 0x00000201);		/* ??? */
-    stl_raw(v ++, 0x00000000);		/* ??? */
+    stl_p(v++, 0x5c623d01);		/* ??? */
+    stl_p(v++, 0x00000201);		/* ??? */
+    stl_p(v++, 0x00000000);		/* ??? */
 
     /* CX3110x WLAN settings */
     ADD_TAG(0x6e0f, 8);
-    stl_raw(v ++, 0x00610025);		/* ??? */
-    stl_raw(v ++, 0xffff0057);		/* ??? */
+    stl_p(v++, 0x00610025);		/* ??? */
+    stl_p(v++, 0xffff0057);		/* ??? */
 
     /* MMC host settings */
     ADD_TAG(0x6e10, 12);
-    stl_raw(v ++, 0xffff000f);		/* ??? */
-    stl_raw(v ++, 0xffffffff);		/* ??? */
-    stl_raw(v ++, 0x00000060);		/* ??? */
+    stl_p(v++, 0xffff000f);		/* ??? */
+    stl_p(v++, 0xffffffff);		/* ??? */
+    stl_p(v++, 0x00000060);		/* ??? */
 
     /* OneNAND chip select */
     ADD_TAG(0x6e11, 10);
-    stl_raw(v ++, 0x00000401);		/* ??? */
-    stl_raw(v ++, 0x0002003a);		/* ??? */
-    stl_raw(v ++, 0x00000002);		/* ??? */
+    stl_p(v++, 0x00000401);		/* ??? */
+    stl_p(v++, 0x0002003a);		/* ??? */
+    stl_p(v++, 0x00000002);		/* ??? */
 
     /* TEA5761 sensor settings */
     ADD_TAG(0x6e12, 2);
-    stl_raw(v ++, 93);			/* GPIO num ??? */
+    stl_p(v++, 93);			/* GPIO num ??? */
 
 #if 0
     /* Unknown tag */
@@ -956,8 +973,8 @@
 #endif
 
     /* End of the list */
-    stl_raw(p ++, 0x00000000);
-    stl_raw(p ++, 0x00000000);
+    stl_p(p++, 0x00000000);
+    stl_p(p++, 0x00000000);
 }
 
 /* This task is normally performed by the bootloader.  If we're loading
@@ -1032,8 +1049,9 @@
     s->mpu->cpu->env.GE = 0x5;
 
     /* If the machine has a slided keyboard, open it */
-    if (s->kbd)
+    if (s->kbd) {
         qemu_irq_raise(qdev_get_gpio_in(s->mpu->gpio, N810_SLIDE_GPIO));
+    }
 }
 
 #define OMAP_TAG_NOKIA_BT	0x4e01
@@ -1119,112 +1137,112 @@
 
     w = p;
 
-    stw_raw(w ++, OMAP_TAG_UART);		/* u16 tag */
-    stw_raw(w ++, 4);				/* u16 len */
-    stw_raw(w ++, (1 << 2) | (1 << 1) | (1 << 0)); /* uint enabled_uarts */
-    w ++;
+    stw_p(w++, OMAP_TAG_UART);			/* u16 tag */
+    stw_p(w++, 4);				/* u16 len */
+    stw_p(w++, (1 << 2) | (1 << 1) | (1 << 0)); /* uint enabled_uarts */
+    w++;
 
 #if 0
-    stw_raw(w ++, OMAP_TAG_SERIAL_CONSOLE);	/* u16 tag */
-    stw_raw(w ++, 4);				/* u16 len */
-    stw_raw(w ++, XLDR_LL_UART + 1);		/* u8 console_uart */
-    stw_raw(w ++, 115200);			/* u32 console_speed */
+    stw_p(w++, OMAP_TAG_SERIAL_CONSOLE);	/* u16 tag */
+    stw_p(w++, 4);				/* u16 len */
+    stw_p(w++, XLDR_LL_UART + 1);		/* u8 console_uart */
+    stw_p(w++, 115200);				/* u32 console_speed */
 #endif
 
-    stw_raw(w ++, OMAP_TAG_LCD);		/* u16 tag */
-    stw_raw(w ++, 36);				/* u16 len */
+    stw_p(w++, OMAP_TAG_LCD);			/* u16 tag */
+    stw_p(w++, 36);				/* u16 len */
     strcpy((void *) w, "QEMU LCD panel");	/* char panel_name[16] */
     w += 8;
     strcpy((void *) w, "blizzard");		/* char ctrl_name[16] */
     w += 8;
-    stw_raw(w ++, N810_BLIZZARD_RESET_GPIO);	/* TODO: n800 s16 nreset_gpio */
-    stw_raw(w ++, 24);				/* u8 data_lines */
+    stw_p(w++, N810_BLIZZARD_RESET_GPIO);	/* TODO: n800 s16 nreset_gpio */
+    stw_p(w++, 24);				/* u8 data_lines */
 
-    stw_raw(w ++, OMAP_TAG_CBUS);		/* u16 tag */
-    stw_raw(w ++, 8);				/* u16 len */
-    stw_raw(w ++, N8X0_CBUS_CLK_GPIO);		/* s16 clk_gpio */
-    stw_raw(w ++, N8X0_CBUS_DAT_GPIO);		/* s16 dat_gpio */
-    stw_raw(w ++, N8X0_CBUS_SEL_GPIO);		/* s16 sel_gpio */
-    w ++;
+    stw_p(w++, OMAP_TAG_CBUS);			/* u16 tag */
+    stw_p(w++, 8);				/* u16 len */
+    stw_p(w++, N8X0_CBUS_CLK_GPIO);		/* s16 clk_gpio */
+    stw_p(w++, N8X0_CBUS_DAT_GPIO);		/* s16 dat_gpio */
+    stw_p(w++, N8X0_CBUS_SEL_GPIO);		/* s16 sel_gpio */
+    w++;
 
-    stw_raw(w ++, OMAP_TAG_EM_ASIC_BB5);	/* u16 tag */
-    stw_raw(w ++, 4);				/* u16 len */
-    stw_raw(w ++, N8X0_RETU_GPIO);		/* s16 retu_irq_gpio */
-    stw_raw(w ++, N8X0_TAHVO_GPIO);		/* s16 tahvo_irq_gpio */
+    stw_p(w++, OMAP_TAG_EM_ASIC_BB5);		/* u16 tag */
+    stw_p(w++, 4);				/* u16 len */
+    stw_p(w++, N8X0_RETU_GPIO);			/* s16 retu_irq_gpio */
+    stw_p(w++, N8X0_TAHVO_GPIO);		/* s16 tahvo_irq_gpio */
 
     gpiosw = (model == 810) ? n810_gpiosw_info : n800_gpiosw_info;
-    for (; gpiosw->name; gpiosw ++) {
-        stw_raw(w ++, OMAP_TAG_GPIO_SWITCH);	/* u16 tag */
-        stw_raw(w ++, 20);			/* u16 len */
+    for (; gpiosw->name; gpiosw++) {
+        stw_p(w++, OMAP_TAG_GPIO_SWITCH);	/* u16 tag */
+        stw_p(w++, 20);				/* u16 len */
         strcpy((void *) w, gpiosw->name);	/* char name[12] */
         w += 6;
-        stw_raw(w ++, gpiosw->line);		/* u16 gpio */
-        stw_raw(w ++, gpiosw->type);
-        stw_raw(w ++, 0);
-        stw_raw(w ++, 0);
+        stw_p(w++, gpiosw->line);		/* u16 gpio */
+        stw_p(w++, gpiosw->type);
+        stw_p(w++, 0);
+        stw_p(w++, 0);
     }
 
-    stw_raw(w ++, OMAP_TAG_NOKIA_BT);		/* u16 tag */
-    stw_raw(w ++, 12);				/* u16 len */
+    stw_p(w++, OMAP_TAG_NOKIA_BT);		/* u16 tag */
+    stw_p(w++, 12);				/* u16 len */
     b = (void *) w;
-    stb_raw(b ++, 0x01);			/* u8 chip_type	(CSR) */
-    stb_raw(b ++, N8X0_BT_WKUP_GPIO);		/* u8 bt_wakeup_gpio */
-    stb_raw(b ++, N8X0_BT_HOST_WKUP_GPIO);	/* u8 host_wakeup_gpio */
-    stb_raw(b ++, N8X0_BT_RESET_GPIO);		/* u8 reset_gpio */
-    stb_raw(b ++, BT_UART + 1);			/* u8 bt_uart */
+    stb_p(b++, 0x01);				/* u8 chip_type	(CSR) */
+    stb_p(b++, N8X0_BT_WKUP_GPIO);		/* u8 bt_wakeup_gpio */
+    stb_p(b++, N8X0_BT_HOST_WKUP_GPIO);		/* u8 host_wakeup_gpio */
+    stb_p(b++, N8X0_BT_RESET_GPIO);		/* u8 reset_gpio */
+    stb_p(b++, BT_UART + 1);			/* u8 bt_uart */
     memcpy(b, &n8x0_bd_addr, 6);		/* u8 bd_addr[6] */
     b += 6;
-    stb_raw(b ++, 0x02);			/* u8 bt_sysclk (38.4) */
+    stb_p(b++, 0x02);				/* u8 bt_sysclk (38.4) */
     w = (void *) b;
 
-    stw_raw(w ++, OMAP_TAG_WLAN_CX3110X);	/* u16 tag */
-    stw_raw(w ++, 8);				/* u16 len */
-    stw_raw(w ++, 0x25);			/* u8 chip_type */
-    stw_raw(w ++, N8X0_WLAN_PWR_GPIO);		/* s16 power_gpio */
-    stw_raw(w ++, N8X0_WLAN_IRQ_GPIO);		/* s16 irq_gpio */
-    stw_raw(w ++, -1);				/* s16 spi_cs_gpio */
+    stw_p(w++, OMAP_TAG_WLAN_CX3110X);		/* u16 tag */
+    stw_p(w++, 8);				/* u16 len */
+    stw_p(w++, 0x25);				/* u8 chip_type */
+    stw_p(w++, N8X0_WLAN_PWR_GPIO);		/* s16 power_gpio */
+    stw_p(w++, N8X0_WLAN_IRQ_GPIO);		/* s16 irq_gpio */
+    stw_p(w++, -1);				/* s16 spi_cs_gpio */
 
-    stw_raw(w ++, OMAP_TAG_MMC);		/* u16 tag */
-    stw_raw(w ++, 16);				/* u16 len */
+    stw_p(w++, OMAP_TAG_MMC);			/* u16 tag */
+    stw_p(w++, 16);				/* u16 len */
     if (model == 810) {
-        stw_raw(w ++, 0x23f);			/* unsigned flags */
-        stw_raw(w ++, -1);			/* s16 power_pin */
-        stw_raw(w ++, -1);			/* s16 switch_pin */
-        stw_raw(w ++, -1);			/* s16 wp_pin */
-        stw_raw(w ++, 0x240);			/* unsigned flags */
-        stw_raw(w ++, 0xc000);			/* s16 power_pin */
-        stw_raw(w ++, 0x0248);			/* s16 switch_pin */
-        stw_raw(w ++, 0xc000);			/* s16 wp_pin */
+        stw_p(w++, 0x23f);			/* unsigned flags */
+        stw_p(w++, -1);				/* s16 power_pin */
+        stw_p(w++, -1);				/* s16 switch_pin */
+        stw_p(w++, -1);				/* s16 wp_pin */
+        stw_p(w++, 0x240);			/* unsigned flags */
+        stw_p(w++, 0xc000);			/* s16 power_pin */
+        stw_p(w++, 0x0248);			/* s16 switch_pin */
+        stw_p(w++, 0xc000);			/* s16 wp_pin */
     } else {
-        stw_raw(w ++, 0xf);			/* unsigned flags */
-        stw_raw(w ++, -1);			/* s16 power_pin */
-        stw_raw(w ++, -1);			/* s16 switch_pin */
-        stw_raw(w ++, -1);			/* s16 wp_pin */
-        stw_raw(w ++, 0);			/* unsigned flags */
-        stw_raw(w ++, 0);			/* s16 power_pin */
-        stw_raw(w ++, 0);			/* s16 switch_pin */
-        stw_raw(w ++, 0);			/* s16 wp_pin */
+        stw_p(w++, 0xf);			/* unsigned flags */
+        stw_p(w++, -1);				/* s16 power_pin */
+        stw_p(w++, -1);				/* s16 switch_pin */
+        stw_p(w++, -1);				/* s16 wp_pin */
+        stw_p(w++, 0);				/* unsigned flags */
+        stw_p(w++, 0);				/* s16 power_pin */
+        stw_p(w++, 0);				/* s16 switch_pin */
+        stw_p(w++, 0);				/* s16 wp_pin */
     }
 
-    stw_raw(w ++, OMAP_TAG_TEA5761);		/* u16 tag */
-    stw_raw(w ++, 4);				/* u16 len */
-    stw_raw(w ++, N8X0_TEA5761_CS_GPIO);	/* u16 enable_gpio */
-    w ++;
+    stw_p(w++, OMAP_TAG_TEA5761);		/* u16 tag */
+    stw_p(w++, 4);				/* u16 len */
+    stw_p(w++, N8X0_TEA5761_CS_GPIO);		/* u16 enable_gpio */
+    w++;
 
     partition = (model == 810) ? n810_part_info : n800_part_info;
-    for (; partition->name; partition ++) {
-        stw_raw(w ++, OMAP_TAG_PARTITION);	/* u16 tag */
-        stw_raw(w ++, 28);			/* u16 len */
+    for (; partition->name; partition++) {
+        stw_p(w++, OMAP_TAG_PARTITION);		/* u16 tag */
+        stw_p(w++, 28);				/* u16 len */
         strcpy((void *) w, partition->name);	/* char name[16] */
         l = (void *) (w + 8);
-        stl_raw(l ++, partition->size);		/* unsigned int size */
-        stl_raw(l ++, partition->offset);	/* unsigned int offset */
-        stl_raw(l ++, partition->mask);		/* unsigned int mask_flags */
+        stl_p(l++, partition->size);		/* unsigned int size */
+        stl_p(l++, partition->offset);		/* unsigned int offset */
+        stl_p(l++, partition->mask);		/* unsigned int mask_flags */
         w = (void *) l;
     }
 
-    stw_raw(w ++, OMAP_TAG_BOOT_REASON);	/* u16 tag */
-    stw_raw(w ++, 12);				/* u16 len */
+    stw_p(w++, OMAP_TAG_BOOT_REASON);		/* u16 tag */
+    stw_p(w++, 12);				/* u16 len */
 #if 0
     strcpy((void *) w, "por");			/* char reason_str[12] */
     strcpy((void *) w, "charger");		/* char reason_str[12] */
@@ -1242,15 +1260,15 @@
     w += 6;
 
     tag = (model == 810) ? "RX-44" : "RX-34";
-    stw_raw(w ++, OMAP_TAG_VERSION_STR);	/* u16 tag */
-    stw_raw(w ++, 24);				/* u16 len */
+    stw_p(w++, OMAP_TAG_VERSION_STR);		/* u16 tag */
+    stw_p(w++, 24);				/* u16 len */
     strcpy((void *) w, "product");		/* char component[12] */
     w += 6;
     strcpy((void *) w, tag);			/* char version[12] */
     w += 6;
 
-    stw_raw(w ++, OMAP_TAG_VERSION_STR);	/* u16 tag */
-    stw_raw(w ++, 24);				/* u16 len */
+    stw_p(w++, OMAP_TAG_VERSION_STR);		/* u16 tag */
+    stw_p(w++, 24);				/* u16 len */
     strcpy((void *) w, "hw-build");		/* char component[12] */
     w += 6;
     strcpy((void *) w, "QEMU ");
@@ -1258,8 +1276,8 @@
     w += 6;
 
     tag = (model == 810) ? "1.1.10-qemu" : "1.1.6-qemu";
-    stw_raw(w ++, OMAP_TAG_VERSION_STR);	/* u16 tag */
-    stw_raw(w ++, 24);				/* u16 len */
+    stw_p(w++, OMAP_TAG_VERSION_STR);		/* u16 tag */
+    stw_p(w++, 24);				/* u16 len */
     strcpy((void *) w, "nolo");			/* char component[12] */
     w += 6;
     strcpy((void *) w, tag);			/* char version[12] */
@@ -1278,14 +1296,14 @@
     return n8x0_atag_setup(p, 810);
 }
 
-static void n8x0_init(QEMUMachineInitArgs *args,
+static void n8x0_init(MachineState *machine,
                       struct arm_boot_info *binfo, int model)
 {
     MemoryRegion *sysmem = get_system_memory();
     struct n800_s *s = (struct n800_s *) g_malloc0(sizeof(*s));
     int sdram_size = binfo->ram_size;
 
-    s->mpu = omap2420_mpu_init(sysmem, sdram_size, args->cpu_model);
+    s->mpu = omap2420_mpu_init(sysmem, sdram_size, machine->cpu_model);
 
     /* Setup peripherals
      *
@@ -1315,9 +1333,9 @@
     n8x0_gpio_setup(s);
     n8x0_nand_setup(s);
     n8x0_i2c_setup(s);
-    if (model == 800)
+    if (model == 800) {
         n800_tsc_kbd_setup(s);
-    else if (model == 810) {
+    } else if (model == 810) {
         n810_tsc_setup(s);
         n810_kbd_setup(s);
     }
@@ -1329,18 +1347,18 @@
         n8x0_usb_setup(s);
     }
 
-    if (args->kernel_filename) {
+    if (machine->kernel_filename) {
         /* Or at the linux loader.  */
-        binfo->kernel_filename = args->kernel_filename;
-        binfo->kernel_cmdline = args->kernel_cmdline;
-        binfo->initrd_filename = args->initrd_filename;
+        binfo->kernel_filename = machine->kernel_filename;
+        binfo->kernel_cmdline = machine->kernel_cmdline;
+        binfo->initrd_filename = machine->initrd_filename;
         arm_load_kernel(s->mpu->cpu, binfo);
 
         qemu_register_reset(n8x0_boot_init, s);
     }
 
     if (option_rom[0].name &&
-        (args->boot_order[0] == 'n' || !args->kernel_filename)) {
+        (machine->boot_order[0] == 'n' || !machine->kernel_filename)) {
         uint8_t nolo_tags[0x10000];
         /* No, wait, better start at the ROM.  */
         s->mpu->cpu->env.regs[15] = OMAP2_Q2_BASE + 0x400000;
@@ -1382,14 +1400,14 @@
     .atag_board = n810_atag_setup,
 };
 
-static void n800_init(QEMUMachineInitArgs *args)
+static void n800_init(MachineState *machine)
 {
-    return n8x0_init(args, &n800_binfo, 800);
+    return n8x0_init(machine, &n800_binfo, 800);
 }
 
-static void n810_init(QEMUMachineInitArgs *args)
+static void n810_init(MachineState *machine)
 {
-    return n8x0_init(args, &n810_binfo, 810);
+    return n8x0_init(machine, &n810_binfo, 810);
 }
 
 static QEMUMachine n800_machine = {
diff --git a/hw/arm/omap_sx1.c b/hw/arm/omap_sx1.c
index 3ba263a..b4f6da6 100644
--- a/hw/arm/omap_sx1.c
+++ b/hw/arm/omap_sx1.c
@@ -98,7 +98,7 @@
     .board_id = 0x265,
 };
 
-static void sx1_init(QEMUMachineInitArgs *args, const int version)
+static void sx1_init(MachineState *machine, const int version)
 {
     struct omap_mpu_state_s *mpu;
     MemoryRegion *address_space = get_system_memory();
@@ -118,7 +118,8 @@
         flash_size = flash2_size;
     }
 
-    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size, args->cpu_model);
+    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size,
+                           machine->cpu_model);
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, NULL, "omap_sx1.flash0-0", flash_size);
@@ -189,29 +190,29 @@
                                 OMAP_CS1_BASE, &cs[1]);
     }
 
-    if (!args->kernel_filename && !fl_idx && !qtest_enabled()) {
+    if (!machine->kernel_filename && !fl_idx && !qtest_enabled()) {
         fprintf(stderr, "Kernel or Flash image must be specified\n");
         exit(1);
     }
 
     /* Load the kernel.  */
-    sx1_binfo.kernel_filename = args->kernel_filename;
-    sx1_binfo.kernel_cmdline = args->kernel_cmdline;
-    sx1_binfo.initrd_filename = args->initrd_filename;
+    sx1_binfo.kernel_filename = machine->kernel_filename;
+    sx1_binfo.kernel_cmdline = machine->kernel_cmdline;
+    sx1_binfo.initrd_filename = machine->initrd_filename;
     arm_load_kernel(mpu->cpu, &sx1_binfo);
 
     /* TODO: fix next line */
     //~ qemu_console_resize(ds, 640, 480);
 }
 
-static void sx1_init_v1(QEMUMachineInitArgs *args)
+static void sx1_init_v1(MachineState *machine)
 {
-    sx1_init(args, 1);
+    sx1_init(machine, 1);
 }
 
-static void sx1_init_v2(QEMUMachineInitArgs *args)
+static void sx1_init_v2(MachineState *machine)
 {
-    sx1_init(args, 2);
+    sx1_init(machine, 2);
 }
 
 static QEMUMachine sx1_machine_v2 = {
diff --git a/hw/arm/palm.c b/hw/arm/palm.c
index fac4f69..e61995f 100644
--- a/hw/arm/palm.c
+++ b/hw/arm/palm.c
@@ -191,12 +191,12 @@
     .board_id = 0x331,
 };
 
-static void palmte_init(QEMUMachineInitArgs *args)
+static void palmte_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     struct omap_mpu_state_s *mpu;
     int flash_size = 0x00800000;
diff --git a/hw/arm/realview.c b/hw/arm/realview.c
index 7e04e50..64b9251 100644
--- a/hw/arm/realview.c
+++ b/hw/arm/realview.c
@@ -45,7 +45,7 @@
     0x76d
 };
 
-static void realview_init(QEMUMachineInitArgs *args,
+static void realview_init(MachineState *machine,
                           enum realview_board_type board_type)
 {
     ARMCPU *cpu = NULL;
@@ -71,7 +71,7 @@
     uint32_t proc_id = 0;
     uint32_t sys_id;
     ram_addr_t low_ram_size;
-    ram_addr_t ram_size = args->ram_size;
+    ram_addr_t ram_size = machine->ram_size;
     hwaddr periphbase = 0;
 
     switch (board_type) {
@@ -91,7 +91,7 @@
         break;
     }
 
-    cpu_oc = cpu_class_by_name(TYPE_ARM_CPU, args->cpu_model);
+    cpu_oc = cpu_class_by_name(TYPE_ARM_CPU, machine->cpu_model);
     if (!cpu_oc) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
@@ -342,45 +342,45 @@
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
     realview_binfo.ram_size = ram_size;
-    realview_binfo.kernel_filename = args->kernel_filename;
-    realview_binfo.kernel_cmdline = args->kernel_cmdline;
-    realview_binfo.initrd_filename = args->initrd_filename;
+    realview_binfo.kernel_filename = machine->kernel_filename;
+    realview_binfo.kernel_cmdline = machine->kernel_cmdline;
+    realview_binfo.initrd_filename = machine->initrd_filename;
     realview_binfo.nb_cpus = smp_cpus;
     realview_binfo.board_id = realview_board_id[board_type];
     realview_binfo.loader_start = (board_type == BOARD_PB_A8 ? 0x70000000 : 0);
     arm_load_kernel(ARM_CPU(first_cpu), &realview_binfo);
 }
 
-static void realview_eb_init(QEMUMachineInitArgs *args)
+static void realview_eb_init(MachineState *machine)
 {
-    if (!args->cpu_model) {
-        args->cpu_model = "arm926";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "arm926";
     }
-    realview_init(args, BOARD_EB);
+    realview_init(machine, BOARD_EB);
 }
 
-static void realview_eb_mpcore_init(QEMUMachineInitArgs *args)
+static void realview_eb_mpcore_init(MachineState *machine)
 {
-    if (!args->cpu_model) {
-        args->cpu_model = "arm11mpcore";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "arm11mpcore";
     }
-    realview_init(args, BOARD_EB_MPCORE);
+    realview_init(machine, BOARD_EB_MPCORE);
 }
 
-static void realview_pb_a8_init(QEMUMachineInitArgs *args)
+static void realview_pb_a8_init(MachineState *machine)
 {
-    if (!args->cpu_model) {
-        args->cpu_model = "cortex-a8";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "cortex-a8";
     }
-    realview_init(args, BOARD_PB_A8);
+    realview_init(machine, BOARD_PB_A8);
 }
 
-static void realview_pbx_a9_init(QEMUMachineInitArgs *args)
+static void realview_pbx_a9_init(MachineState *machine)
 {
-    if (!args->cpu_model) {
-        args->cpu_model = "cortex-a9";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "cortex-a9";
     }
-    realview_init(args, BOARD_PBX_A9);
+    realview_init(machine, BOARD_PBX_A9);
 }
 
 static QEMUMachine realview_eb_machine = {
diff --git a/hw/arm/spitz.c b/hw/arm/spitz.c
index a179c1d..5455dbf 100644
--- a/hw/arm/spitz.c
+++ b/hw/arm/spitz.c
@@ -887,14 +887,14 @@
     .ram_size = 0x04000000,
 };
 
-static void spitz_common_init(QEMUMachineInitArgs *args,
+static void spitz_common_init(MachineState *machine,
                               enum spitz_model_e model, int arm_id)
 {
     PXA2xxState *mpu;
     DeviceState *scp0, *scp1 = NULL;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
 
     if (!cpu_model)
         cpu_model = (model == terrier) ? "pxa270-c5" : "pxa270-c0";
@@ -935,32 +935,32 @@
         /* A 4.0 GB microdrive is permanently sitting in CF slot 0.  */
         spitz_microdrive_attach(mpu, 0);
 
-    spitz_binfo.kernel_filename = args->kernel_filename;
-    spitz_binfo.kernel_cmdline = args->kernel_cmdline;
-    spitz_binfo.initrd_filename = args->initrd_filename;
+    spitz_binfo.kernel_filename = machine->kernel_filename;
+    spitz_binfo.kernel_cmdline = machine->kernel_cmdline;
+    spitz_binfo.initrd_filename = machine->initrd_filename;
     spitz_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &spitz_binfo);
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static void spitz_init(QEMUMachineInitArgs *args)
+static void spitz_init(MachineState *machine)
 {
-    spitz_common_init(args, spitz, 0x2c9);
+    spitz_common_init(machine, spitz, 0x2c9);
 }
 
-static void borzoi_init(QEMUMachineInitArgs *args)
+static void borzoi_init(MachineState *machine)
 {
-    spitz_common_init(args, borzoi, 0x33f);
+    spitz_common_init(machine, borzoi, 0x33f);
 }
 
-static void akita_init(QEMUMachineInitArgs *args)
+static void akita_init(MachineState *machine)
 {
-    spitz_common_init(args, akita, 0x2e8);
+    spitz_common_init(machine, akita, 0x2e8);
 }
 
-static void terrier_init(QEMUMachineInitArgs *args)
+static void terrier_init(MachineState *machine)
 {
-    spitz_common_init(args, terrier, 0x33f);
+    spitz_common_init(machine, terrier, 0x33f);
 }
 
 static QEMUMachine akitapda_machine = {
diff --git a/hw/arm/stellaris.c b/hw/arm/stellaris.c
index a2095c0..80028e8 100644
--- a/hw/arm/stellaris.c
+++ b/hw/arm/stellaris.c
@@ -1290,9 +1290,10 @@
 
             sddev = ssi_create_slave(bus, "ssi-sd");
             ssddev = ssi_create_slave(bus, "ssd0323");
-            gpio_out[GPIO_D][0] = qemu_irq_split(qdev_get_gpio_in(sddev, 0),
-                                                 qdev_get_gpio_in(ssddev, 0));
-            gpio_out[GPIO_C][7] = qdev_get_gpio_in(ssddev, 1);
+            gpio_out[GPIO_D][0] = qemu_irq_split(
+                    qdev_get_gpio_in_named(sddev, SSI_GPIO_CS, 0),
+                    qdev_get_gpio_in_named(ssddev, SSI_GPIO_CS, 0));
+            gpio_out[GPIO_C][7] = qdev_get_gpio_in(ssddev, 0);
 
             /* Make sure the select pin is high.  */
             qemu_irq_raise(gpio_out[GPIO_D][0]);
@@ -1333,17 +1334,17 @@
 }
 
 /* FIXME: Figure out how to generate these from stellaris_boards.  */
-static void lm3s811evb_init(QEMUMachineInitArgs *args)
+static void lm3s811evb_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[0]);
 }
 
-static void lm3s6965evb_init(QEMUMachineInitArgs *args)
+static void lm3s6965evb_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
diff --git a/hw/arm/tosa.c b/hw/arm/tosa.c
index 2069f55..abc0f2a 100644
--- a/hw/arm/tosa.c
+++ b/hw/arm/tosa.c
@@ -211,12 +211,12 @@
     .ram_size = 0x04000000,
 };
 
-static void tosa_init(QEMUMachineInitArgs *args)
+static void tosa_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
     PXA2xxState *mpu;
diff --git a/hw/arm/versatilepb.c b/hw/arm/versatilepb.c
index e5493b4..dea5fc7 100644
--- a/hw/arm/versatilepb.c
+++ b/hw/arm/versatilepb.c
@@ -173,7 +173,7 @@
 
 static struct arm_boot_info versatile_binfo;
 
-static void versatile_init(QEMUMachineInitArgs *args, int board_id)
+static void versatile_init(MachineState *machine, int board_id)
 {
     ARMCPU *cpu;
     MemoryRegion *sysmem = get_system_memory();
@@ -190,15 +190,15 @@
     int done_smc = 0;
     DriveInfo *dinfo;
 
-    if (!args->cpu_model) {
-        args->cpu_model = "arm926";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "arm926";
     }
-    cpu = cpu_arm_init(args->cpu_model);
+    cpu = cpu_arm_init(machine->cpu_model);
     if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    memory_region_init_ram(ram, NULL, "versatile.ram", args->ram_size);
+    memory_region_init_ram(ram, NULL, "versatile.ram", machine->ram_size);
     vmstate_register_ram_global(ram);
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero.  */
@@ -344,22 +344,22 @@
         fprintf(stderr, "qemu: Error registering flash memory.\n");
     }
 
-    versatile_binfo.ram_size = args->ram_size;
-    versatile_binfo.kernel_filename = args->kernel_filename;
-    versatile_binfo.kernel_cmdline = args->kernel_cmdline;
-    versatile_binfo.initrd_filename = args->initrd_filename;
+    versatile_binfo.ram_size = machine->ram_size;
+    versatile_binfo.kernel_filename = machine->kernel_filename;
+    versatile_binfo.kernel_cmdline = machine->kernel_cmdline;
+    versatile_binfo.initrd_filename = machine->initrd_filename;
     versatile_binfo.board_id = board_id;
     arm_load_kernel(cpu, &versatile_binfo);
 }
 
-static void vpb_init(QEMUMachineInitArgs *args)
+static void vpb_init(MachineState *machine)
 {
-    versatile_init(args, 0x183);
+    versatile_init(machine, 0x183);
 }
 
-static void vab_init(QEMUMachineInitArgs *args)
+static void vab_init(MachineState *machine)
 {
-    versatile_init(args, 0x25e);
+    versatile_init(machine, 0x25e);
 }
 
 static QEMUMachine versatilepb_machine = {
diff --git a/hw/arm/vexpress.c b/hw/arm/vexpress.c
index 169eb06..33ff422 100644
--- a/hw/arm/vexpress.c
+++ b/hw/arm/vexpress.c
@@ -509,7 +509,7 @@
 }
 
 static void vexpress_common_init(VEDBoardInfo *daughterboard,
-                                 QEMUMachineInitArgs *args)
+                                 MachineState *machine)
 {
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
@@ -525,7 +525,8 @@
     const hwaddr *map = daughterboard->motherboard_map;
     int i;
 
-    daughterboard->init(daughterboard, args->ram_size, args->cpu_model, pic);
+    daughterboard->init(daughterboard, machine->ram_size, machine->cpu_model,
+                        pic);
 
     /* Motherboard peripherals: the wiring is the same but the
      * addresses vary between the legacy and A-Series memory maps.
@@ -639,10 +640,10 @@
                              pic[40 + i]);
     }
 
-    daughterboard->bootinfo.ram_size = args->ram_size;
-    daughterboard->bootinfo.kernel_filename = args->kernel_filename;
-    daughterboard->bootinfo.kernel_cmdline = args->kernel_cmdline;
-    daughterboard->bootinfo.initrd_filename = args->initrd_filename;
+    daughterboard->bootinfo.ram_size = machine->ram_size;
+    daughterboard->bootinfo.kernel_filename = machine->kernel_filename;
+    daughterboard->bootinfo.kernel_cmdline = machine->kernel_cmdline;
+    daughterboard->bootinfo.initrd_filename = machine->initrd_filename;
     daughterboard->bootinfo.nb_cpus = smp_cpus;
     daughterboard->bootinfo.board_id = VEXPRESS_BOARD_ID;
     daughterboard->bootinfo.loader_start = daughterboard->loader_start;
@@ -653,14 +654,14 @@
     arm_load_kernel(ARM_CPU(first_cpu), &daughterboard->bootinfo);
 }
 
-static void vexpress_a9_init(QEMUMachineInitArgs *args)
+static void vexpress_a9_init(MachineState *machine)
 {
-    vexpress_common_init(&a9_daughterboard, args);
+    vexpress_common_init(&a9_daughterboard, machine);
 }
 
-static void vexpress_a15_init(QEMUMachineInitArgs *args)
+static void vexpress_a15_init(MachineState *machine)
 {
-    vexpress_common_init(&a15_daughterboard, args);
+    vexpress_common_init(&a15_daughterboard, machine);
 }
 
 static QEMUMachine vexpress_a9_machine = {
diff --git a/hw/arm/virt.c b/hw/arm/virt.c
index ea4f02d..3b55a4b 100644
--- a/hw/arm/virt.c
+++ b/hw/arm/virt.c
@@ -383,13 +383,13 @@
     return board->fdt;
 }
 
-static void machvirt_init(QEMUMachineInitArgs *args)
+static void machvirt_init(MachineState *machine)
 {
     qemu_irq pic[NUM_IRQS];
     MemoryRegion *sysmem = get_system_memory();
     int n;
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
     VirtBoardInfo *vbi;
 
     if (!cpu_model) {
@@ -415,7 +415,7 @@
         exit(1);
     }
 
-    if (args->ram_size > vbi->memmap[VIRT_MEM].size) {
+    if (machine->ram_size > vbi->memmap[VIRT_MEM].size) {
         error_report("mach-virt: cannot model more than 30GB RAM");
         exit(1);
     }
@@ -447,7 +447,7 @@
     }
     fdt_add_cpu_nodes(vbi);
 
-    memory_region_init_ram(ram, NULL, "mach-virt.ram", args->ram_size);
+    memory_region_init_ram(ram, NULL, "mach-virt.ram", machine->ram_size);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(sysmem, vbi->memmap[VIRT_MEM].base, ram);
 
@@ -461,10 +461,10 @@
      */
     create_virtio_devices(vbi, pic);
 
-    vbi->bootinfo.ram_size = args->ram_size;
-    vbi->bootinfo.kernel_filename = args->kernel_filename;
-    vbi->bootinfo.kernel_cmdline = args->kernel_cmdline;
-    vbi->bootinfo.initrd_filename = args->initrd_filename;
+    vbi->bootinfo.ram_size = machine->ram_size;
+    vbi->bootinfo.kernel_filename = machine->kernel_filename;
+    vbi->bootinfo.kernel_cmdline = machine->kernel_cmdline;
+    vbi->bootinfo.initrd_filename = machine->initrd_filename;
     vbi->bootinfo.nb_cpus = smp_cpus;
     vbi->bootinfo.board_id = -1;
     vbi->bootinfo.loader_start = vbi->memmap[VIRT_MEM].base;
diff --git a/hw/arm/xilinx_zynq.c b/hw/arm/xilinx_zynq.c
index 9ee21e7..ba5aa82 100644
--- a/hw/arm/xilinx_zynq.c
+++ b/hw/arm/xilinx_zynq.c
@@ -94,20 +94,20 @@
         for (j = 0; j < num_ss; ++j) {
             flash_dev = ssi_create_slave(spi, "n25q128");
 
-            cs_line = qdev_get_gpio_in(flash_dev, 0);
+            cs_line = qdev_get_gpio_in_named(flash_dev, SSI_GPIO_CS, 0);
             sysbus_connect_irq(busdev, i * num_ss + j + 1, cs_line);
         }
     }
 
 }
 
-static void zynq_init(QEMUMachineInitArgs *args)
+static void zynq_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     ObjectClass *cpu_oc;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
diff --git a/hw/arm/z2.c b/hw/arm/z2.c
index 5df014b..ab9e4c9 100644
--- a/hw/arm/z2.c
+++ b/hw/arm/z2.c
@@ -300,12 +300,12 @@
     .class_init    = aer915_class_init,
 };
 
-static void z2_init(QEMUMachineInitArgs *args)
+static void z2_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     uint32_t sector_len = 0x10000;
     PXA2xxState *mpu;
diff --git a/hw/audio/intel-hda.c b/hw/audio/intel-hda.c
index d41f82c..3cfb66c 100644
--- a/hw/audio/intel-hda.c
+++ b/hw/audio/intel-hda.c
@@ -245,7 +245,7 @@
 
     /* update global status */
     if (sts & d->int_ctl) {
-        sts |= (1 << 31);
+        sts |= (1U << 31);
     }
 
     d->int_sts = sts;
@@ -257,7 +257,7 @@
     int level;
 
     intel_hda_update_int_sts(d);
-    if (d->int_sts & (1 << 31) && d->int_ctl & (1 << 31)) {
+    if (d->int_sts & (1U << 31) && d->int_ctl & (1U << 31)) {
         level = 1;
     } else {
         level = 0;
@@ -574,7 +574,7 @@
     if (st->ctl & 0x01) {
         /* reset */
         dprint(d, 1, "st #%d: reset\n", reg->stream);
-        st->ctl = 0;
+        st->ctl = SD_STS_FIFO_READY << 24;
     }
     if ((st->ctl & 0x02) != (old & 0x02)) {
         uint32_t stnr = (st->ctl >> 20) & 0x0f;
@@ -829,6 +829,7 @@
         .wclear   = 0x1c000000,                                       \
         .offset   = offsetof(IntelHDAState, st[_i].ctl),              \
         .whandler = intel_hda_set_st_ctl,                             \
+        .reset    = SD_STS_FIFO_READY << 24                           \
     },                                                                \
     [ ST_REG(_i, ICH6_REG_SD_LPIB) ] = {                              \
         .stream   = _i,                                               \
diff --git a/hw/block/dataplane/virtio-blk.c b/hw/block/dataplane/virtio-blk.c
index 70b8a5a..e49c253 100644
--- a/hw/block/dataplane/virtio-blk.c
+++ b/hw/block/dataplane/virtio-blk.c
@@ -70,6 +70,9 @@
                                              queue */
 
     unsigned int num_reqs;
+
+    /* Operation blocker on BDS */
+    Error *blocker;
 };
 
 /* Raise an interrupt to signal guest, if necessary */
@@ -350,6 +353,7 @@
 {
     VirtIOBlockDataPlane *s;
     int fd;
+    Error *local_err = NULL;
 
     *dataplane = NULL;
 
@@ -372,9 +376,10 @@
     /* If dataplane is (re-)enabled while the guest is running there could be
      * block jobs that can conflict.
      */
-    if (bdrv_in_use(blk->conf.bs)) {
-        error_setg(errp,
-                   "cannot start dataplane thread while device is in use");
+    if (bdrv_op_is_blocked(blk->conf.bs, BLOCK_OP_TYPE_DATAPLANE, &local_err)) {
+        error_report("cannot start dataplane thread: %s",
+                      error_get_pretty(local_err));
+        error_free(local_err);
         return;
     }
 
@@ -406,8 +411,8 @@
     }
     s->ctx = iothread_get_aio_context(s->iothread);
 
-    /* Prevent block operations that conflict with data plane thread */
-    bdrv_set_in_use(blk->conf.bs, 1);
+    error_setg(&s->blocker, "block device is in use by data plane");
+    bdrv_op_block_all(blk->conf.bs, s->blocker);
 
     *dataplane = s;
 }
@@ -420,7 +425,8 @@
     }
 
     virtio_blk_data_plane_stop(s);
-    bdrv_set_in_use(s->blk->conf.bs, 0);
+    bdrv_op_unblock_all(s->blk->conf.bs, s->blocker);
+    error_free(s->blocker);
     object_unref(OBJECT(s->iothread));
     g_free(s);
 }
diff --git a/hw/block/virtio-blk.c b/hw/block/virtio-blk.c
index 8a568e5..b1fc1de 100644
--- a/hw/block/virtio-blk.c
+++ b/hw/block/virtio-blk.c
@@ -487,12 +487,12 @@
 
     bdrv_get_geometry(s->bs, &capacity);
     memset(&blkcfg, 0, sizeof(blkcfg));
-    stq_raw(&blkcfg.capacity, capacity);
-    stl_raw(&blkcfg.seg_max, 128 - 2);
-    stw_raw(&blkcfg.cylinders, s->conf->cyls);
-    stl_raw(&blkcfg.blk_size, blk_size);
-    stw_raw(&blkcfg.min_io_size, s->conf->min_io_size / blk_size);
-    stw_raw(&blkcfg.opt_io_size, s->conf->opt_io_size / blk_size);
+    stq_p(&blkcfg.capacity, capacity);
+    stl_p(&blkcfg.seg_max, 128 - 2);
+    stw_p(&blkcfg.cylinders, s->conf->cyls);
+    stl_p(&blkcfg.blk_size, blk_size);
+    stw_p(&blkcfg.min_io_size, s->conf->min_io_size / blk_size);
+    stw_p(&blkcfg.opt_io_size, s->conf->opt_io_size / blk_size);
     blkcfg.heads = s->conf->heads;
     /*
      * We must ensure that the block device capacity is a multiple of
diff --git a/hw/core/machine.c b/hw/core/machine.c
index d3ffef7..cbba679 100644
--- a/hw/core/machine.c
+++ b/hw/core/machine.c
@@ -11,6 +11,284 @@
  */
 
 #include "hw/boards.h"
+#include "qapi/visitor.h"
+
+static char *machine_get_accel(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->accel);
+}
+
+static void machine_set_accel(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->accel = g_strdup(value);
+}
+
+static bool machine_get_kernel_irqchip(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->kernel_irqchip;
+}
+
+static void machine_set_kernel_irqchip(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->kernel_irqchip = value;
+}
+
+static void machine_get_kvm_shadow_mem(Object *obj, Visitor *v,
+                                       void *opaque, const char *name,
+                                       Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+    int64_t value = ms->kvm_shadow_mem;
+
+    visit_type_int(v, &value, name, errp);
+}
+
+static void machine_set_kvm_shadow_mem(Object *obj, Visitor *v,
+                                       void *opaque, const char *name,
+                                       Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+    Error *error = NULL;
+    int64_t value;
+
+    visit_type_int(v, &value, name, &error);
+    if (error) {
+        error_propagate(errp, error);
+        return;
+    }
+
+    ms->kvm_shadow_mem = value;
+}
+
+static char *machine_get_kernel(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->kernel_filename);
+}
+
+static void machine_set_kernel(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->kernel_filename = g_strdup(value);
+}
+
+static char *machine_get_initrd(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->initrd_filename);
+}
+
+static void machine_set_initrd(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->initrd_filename = g_strdup(value);
+}
+
+static char *machine_get_append(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->kernel_cmdline);
+}
+
+static void machine_set_append(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->kernel_cmdline = g_strdup(value);
+}
+
+static char *machine_get_dtb(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->dtb);
+}
+
+static void machine_set_dtb(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->dtb = g_strdup(value);
+}
+
+static char *machine_get_dumpdtb(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->dumpdtb);
+}
+
+static void machine_set_dumpdtb(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->dumpdtb = g_strdup(value);
+}
+
+static void machine_get_phandle_start(Object *obj, Visitor *v,
+                                       void *opaque, const char *name,
+                                       Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+    int64_t value = ms->phandle_start;
+
+    visit_type_int(v, &value, name, errp);
+}
+
+static void machine_set_phandle_start(Object *obj, Visitor *v,
+                                       void *opaque, const char *name,
+                                       Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+    Error *error = NULL;
+    int64_t value;
+
+    visit_type_int(v, &value, name, &error);
+    if (error) {
+        error_propagate(errp, error);
+        return;
+    }
+
+    ms->phandle_start = value;
+}
+
+static char *machine_get_dt_compatible(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->dt_compatible);
+}
+
+static void machine_set_dt_compatible(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->dt_compatible = g_strdup(value);
+}
+
+static bool machine_get_dump_guest_core(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->dump_guest_core;
+}
+
+static void machine_set_dump_guest_core(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->dump_guest_core = value;
+}
+
+static bool machine_get_mem_merge(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->mem_merge;
+}
+
+static void machine_set_mem_merge(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->mem_merge = value;
+}
+
+static bool machine_get_usb(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return ms->usb;
+}
+
+static void machine_set_usb(Object *obj, bool value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->usb = value;
+}
+
+static char *machine_get_firmware(Object *obj, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    return g_strdup(ms->firmware);
+}
+
+static void machine_set_firmware(Object *obj, const char *value, Error **errp)
+{
+    MachineState *ms = MACHINE(obj);
+
+    ms->firmware = g_strdup(value);
+}
+
+static void machine_initfn(Object *obj)
+{
+    object_property_add_str(obj, "accel",
+                            machine_get_accel, machine_set_accel, NULL);
+    object_property_add_bool(obj, "kernel_irqchip",
+                             machine_get_kernel_irqchip,
+                             machine_set_kernel_irqchip,
+                             NULL);
+    object_property_add(obj, "kvm_shadow_mem", "int",
+                        machine_get_kvm_shadow_mem,
+                        machine_set_kvm_shadow_mem,
+                        NULL, NULL, NULL);
+    object_property_add_str(obj, "kernel",
+                            machine_get_kernel, machine_set_kernel, NULL);
+    object_property_add_str(obj, "initrd",
+                            machine_get_initrd, machine_set_initrd, NULL);
+    object_property_add_str(obj, "append",
+                            machine_get_append, machine_set_append, NULL);
+    object_property_add_str(obj, "dtb",
+                            machine_get_dtb, machine_set_dtb, NULL);
+    object_property_add_str(obj, "dumpdtb",
+                            machine_get_dumpdtb, machine_set_dumpdtb, NULL);
+    object_property_add(obj, "phandle_start", "int",
+                        machine_get_phandle_start,
+                        machine_set_phandle_start,
+                        NULL, NULL, NULL);
+    object_property_add_str(obj, "dt_compatible",
+                            machine_get_dt_compatible,
+                            machine_set_dt_compatible,
+                            NULL);
+    object_property_add_bool(obj, "dump-guest-core",
+                             machine_get_dump_guest_core,
+                             machine_set_dump_guest_core,
+                             NULL);
+    object_property_add_bool(obj, "mem-merge",
+                             machine_get_mem_merge, machine_set_mem_merge, NULL);
+    object_property_add_bool(obj, "usb", machine_get_usb, machine_set_usb, NULL);
+    object_property_add_str(obj, "firmware",
+                            machine_get_firmware, machine_set_firmware, NULL);
+}
+
+static void machine_finalize(Object *obj)
+{
+    MachineState *ms = MACHINE(obj);
+
+    g_free(ms->accel);
+    g_free(ms->kernel_filename);
+    g_free(ms->initrd_filename);
+    g_free(ms->kernel_cmdline);
+    g_free(ms->dtb);
+    g_free(ms->dumpdtb);
+    g_free(ms->dt_compatible);
+    g_free(ms->firmware);
+}
 
 static const TypeInfo machine_info = {
     .name = TYPE_MACHINE,
@@ -18,6 +296,8 @@
     .abstract = true,
     .class_size = sizeof(MachineClass),
     .instance_size = sizeof(MachineState),
+    .instance_init = machine_initfn,
+    .instance_finalize = machine_finalize,
 };
 
 static void machine_register_types(void)
diff --git a/hw/core/null-machine.c b/hw/core/null-machine.c
index d813c08..1ec7c3b 100644
--- a/hw/core/null-machine.c
+++ b/hw/core/null-machine.c
@@ -15,7 +15,7 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
 
-static void machine_none_init(QEMUMachineInitArgs *args)
+static void machine_none_init(MachineState *machine)
 {
 }
 
diff --git a/hw/core/qdev.c b/hw/core/qdev.c
index 936eae6..e65a5aa 100644
--- a/hw/core/qdev.c
+++ b/hw/core/qdev.c
@@ -312,30 +312,82 @@
     return dev->parent_bus;
 }
 
+static NamedGPIOList *qdev_get_named_gpio_list(DeviceState *dev,
+                                               const char *name)
+{
+    NamedGPIOList *ngl;
+
+    QLIST_FOREACH(ngl, &dev->gpios, node) {
+        /* NULL is a valid and matchable name, otherwise do a normal
+         * strcmp match.
+         */
+        if ((!ngl->name && !name) ||
+                (name && ngl->name && strcmp(name, ngl->name) == 0)) {
+            return ngl;
+        }
+    }
+
+    ngl = g_malloc0(sizeof(*ngl));
+    ngl->name = g_strdup(name);
+    QLIST_INSERT_HEAD(&dev->gpios, ngl, node);
+    return ngl;
+}
+
+void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
+                             const char *name, int n)
+{
+    NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+
+    gpio_list->in = qemu_extend_irqs(gpio_list->in, gpio_list->num_in, handler,
+                                     dev, n);
+    gpio_list->num_in += n;
+}
+
 void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n)
 {
-    dev->gpio_in = qemu_extend_irqs(dev->gpio_in, dev->num_gpio_in, handler,
-                                        dev, n);
-    dev->num_gpio_in += n;
+    qdev_init_gpio_in_named(dev, handler, NULL, n);
+}
+
+void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
+                              const char *name, int n)
+{
+    NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+
+    assert(gpio_list->num_out == 0);
+    gpio_list->num_out = n;
+    gpio_list->out = pins;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
 {
-    assert(dev->num_gpio_out == 0);
-    dev->num_gpio_out = n;
-    dev->gpio_out = pins;
+    qdev_init_gpio_out_named(dev, pins, NULL, n);
+}
+
+qemu_irq qdev_get_gpio_in_named(DeviceState *dev, const char *name, int n)
+{
+    NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+
+    assert(n >= 0 && n < gpio_list->num_in);
+    return gpio_list->in[n];
 }
 
 qemu_irq qdev_get_gpio_in(DeviceState *dev, int n)
 {
-    assert(n >= 0 && n < dev->num_gpio_in);
-    return dev->gpio_in[n];
+    return qdev_get_gpio_in_named(dev, NULL, n);
+}
+
+void qdev_connect_gpio_out_named(DeviceState *dev, const char *name, int n,
+                                 qemu_irq pin)
+{
+    NamedGPIOList *gpio_list = qdev_get_named_gpio_list(dev, name);
+
+    assert(n >= 0 && n < gpio_list->num_out);
+    gpio_list->out[n] = pin;
 }
 
 void qdev_connect_gpio_out(DeviceState * dev, int n, qemu_irq pin)
 {
-    assert(n >= 0 && n < dev->num_gpio_out);
-    dev->gpio_out[n] = pin;
+    qdev_connect_gpio_out_named(dev, NULL, n, pin);
 }
 
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name)
@@ -844,6 +896,7 @@
     object_property_add_link(OBJECT(dev), "parent_bus", TYPE_BUS,
                              (Object **)&dev->parent_bus, NULL, 0,
                              &error_abort);
+    QLIST_INIT(&dev->gpios);
 }
 
 static void device_post_init(Object *obj)
@@ -854,10 +907,22 @@
 /* Unlink device from bus and free the structure.  */
 static void device_finalize(Object *obj)
 {
+    NamedGPIOList *ngl, *next;
+
     DeviceState *dev = DEVICE(obj);
     if (dev->opts) {
         qemu_opts_del(dev->opts);
     }
+
+    QLIST_FOREACH_SAFE(ngl, &dev->gpios, node, next) {
+        QLIST_REMOVE(ngl, node);
+        qemu_free_irqs(ngl->in);
+        g_free(ngl->name);
+        g_free(ngl);
+        /* ngl->out irqs are owned by the other end and should not be freed
+         * here
+         */
+    }
 }
 
 static void device_class_base_init(ObjectClass *class, void *data)
diff --git a/hw/cris/axis_dev88.c b/hw/cris/axis_dev88.c
index 645e45c..1849338 100644
--- a/hw/cris/axis_dev88.c
+++ b/hw/cris/axis_dev88.c
@@ -243,12 +243,12 @@
 static struct cris_load_info li;
 
 static
-void axisdev88_init(QEMUMachineInitArgs *args)
+void axisdev88_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
     CRISCPU *cpu;
     CPUCRISState *env;
     DeviceState *dev;
diff --git a/hw/display/jazz_led.c b/hw/display/jazz_led.c
index e9bb005..12b1707 100644
--- a/hw/display/jazz_led.c
+++ b/hw/display/jazz_led.c
@@ -173,6 +173,7 @@
             case 16:
                 color_segment = rgb_to_pixel16(0xaa, 0xaa, 0xaa);
                 color_led = rgb_to_pixel16(0x00, 0xff, 0x00);
+                break;
             case 24:
                 color_segment = rgb_to_pixel24(0xaa, 0xaa, 0xaa);
                 color_led = rgb_to_pixel24(0x00, 0xff, 0x00);
diff --git a/hw/display/omap_lcd_template.h b/hw/display/omap_lcd_template.h
index 2fb96f8..e5dd447 100644
--- a/hw/display/omap_lcd_template.h
+++ b/hw/display/omap_lcd_template.h
@@ -50,7 +50,7 @@
     uint8_t v, r, g, b;
 
     do {
-        v = ldub_raw((void *) s);
+        v = ldub_p((void *) s);
         r = (pal[v & 3] >> 4) & 0xf0;
         g = pal[v & 3] & 0xf0;
         b = (pal[v & 3] << 4) & 0xf0;
@@ -89,7 +89,7 @@
     uint8_t v, r, g, b;
 
     do {
-        v = ldub_raw((void *) s);
+        v = ldub_p((void *) s);
         r = (pal[v & 0xf] >> 4) & 0xf0;
         g = pal[v & 0xf] & 0xf0;
         b = (pal[v & 0xf] << 4) & 0xf0;
@@ -116,7 +116,7 @@
     uint8_t v, r, g, b;
 
     do {
-        v = ldub_raw((void *) s);
+        v = ldub_p((void *) s);
         r = (pal[v] >> 4) & 0xf0;
         g = pal[v] & 0xf0;
         b = (pal[v] << 4) & 0xf0;
@@ -136,7 +136,7 @@
     uint8_t r, g, b;
 
     do {
-        v = lduw_raw((void *) s);
+        v = lduw_p((void *) s);
         r = (v >> 4) & 0xf0;
         g = v & 0xf0;
         b = (v << 4) & 0xf0;
@@ -159,7 +159,7 @@
     uint8_t r, g, b;
 
     do {
-        v = lduw_raw((void *) s);
+        v = lduw_p((void *) s);
         r = (v >> 8) & 0xf8;
         g = (v >> 3) & 0xfc;
         b = (v << 3) & 0xf8;
diff --git a/hw/display/pxa2xx_lcd.c b/hw/display/pxa2xx_lcd.c
index 80edb70..611fb17 100644
--- a/hw/display/pxa2xx_lcd.c
+++ b/hw/display/pxa2xx_lcd.c
@@ -620,17 +620,6 @@
             src += 2;
             break;
         case 1: /* 16 bpp plus transparency */
-            alpha = *(uint16_t *) src & (1 << 24);
-            if (s->control[0] & LCCR0_CMS)
-                r = g = b = *(uint16_t *) src & 0xff;
-            else {
-                r = (*(uint16_t *) src & 0xf800) >> 8;
-                g = (*(uint16_t *) src & 0x07e0) >> 3;
-                b = (*(uint16_t *) src & 0x001f) << 3;
-            }
-            src += 2;
-            break;
-        case 2: /* 18 bpp plus transparency */
             alpha = *(uint32_t *) src & (1 << 24);
             if (s->control[0] & LCCR0_CMS)
                 r = g = b = *(uint32_t *) src & 0xff;
@@ -641,6 +630,17 @@
             }
             src += 4;
             break;
+        case 2: /* 18 bpp plus transparency */
+            alpha = *(uint32_t *) src & (1 << 24);
+            if (s->control[0] & LCCR0_CMS)
+                r = g = b = *(uint32_t *) src & 0xff;
+            else {
+                r = (*(uint32_t *) src & 0xfc0000) >> 16;
+                g = (*(uint32_t *) src & 0x00fc00) >> 8;
+                b = (*(uint32_t *) src & 0x0000fc);
+            }
+            src += 4;
+            break;
         case 3: /* 24 bpp plus transparency */
             alpha = *(uint32_t *) src & (1 << 24);
             if (s->control[0] & LCCR0_CMS)
diff --git a/hw/display/sm501_template.h b/hw/display/sm501_template.h
index d4cea9e..f33e499 100644
--- a/hw/display/sm501_template.h
+++ b/hw/display/sm501_template.h
@@ -47,7 +47,7 @@
 {
     uint8_t v, r, g, b;
     do {
-      	v = ldub_raw(s);
+	v = ldub_p(s);
 	r = (pal[v] >> 16) & 0xff;
 	g = (pal[v] >>  8) & 0xff;
 	b = (pal[v] >>  0) & 0xff;
@@ -64,7 +64,7 @@
     uint8_t r, g, b;
 
     do {
-	rgb565 = lduw_raw(s);
+	rgb565 = lduw_p(s);
 	r = ((rgb565 >> 11) & 0x1f) << 3;
 	g = ((rgb565 >>  5) & 0x3f) << 2;
 	b = ((rgb565 >>  0) & 0x1f) << 3;
@@ -80,7 +80,7 @@
     uint8_t r, g, b;
 
     do {
-	ldub_raw(s);
+	ldub_p(s);
 #if defined(TARGET_WORDS_BIGENDIAN)
         r = s[1];
         g = s[2];
diff --git a/hw/display/vga_template.h b/hw/display/vga_template.h
index 6cfae56..90ec9c2 100644
--- a/hw/display/vga_template.h
+++ b/hw/display/vga_template.h
@@ -361,7 +361,7 @@
 
     w = width;
     do {
-        v = lduw_raw((void *)s);
+        v = lduw_p((void *)s);
         r = (v >> 7) & 0xf8;
         g = (v >> 2) & 0xf8;
         b = (v << 3) & 0xf8;
@@ -386,7 +386,7 @@
 
     w = width;
     do {
-        v = lduw_raw((void *)s);
+        v = lduw_p((void *)s);
         r = (v >> 8) & 0xf8;
         g = (v >> 3) & 0xfc;
         b = (v << 3) & 0xf8;
diff --git a/hw/i386/kvm/clock.c b/hw/i386/kvm/clock.c
index 892aa02..bef2504 100644
--- a/hw/i386/kvm/clock.c
+++ b/hw/i386/kvm/clock.c
@@ -14,8 +14,10 @@
  */
 
 #include "qemu-common.h"
+#include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "sysemu/kvm.h"
+#include "sysemu/cpus.h"
 #include "hw/sysbus.h"
 #include "hw/kvm/clock.h"
 
@@ -34,6 +36,48 @@
     bool clock_valid;
 } KVMClockState;
 
+struct pvclock_vcpu_time_info {
+    uint32_t   version;
+    uint32_t   pad0;
+    uint64_t   tsc_timestamp;
+    uint64_t   system_time;
+    uint32_t   tsc_to_system_mul;
+    int8_t     tsc_shift;
+    uint8_t    flags;
+    uint8_t    pad[2];
+} __attribute__((__packed__)); /* 32 bytes */
+
+static uint64_t kvmclock_current_nsec(KVMClockState *s)
+{
+    CPUState *cpu = first_cpu;
+    CPUX86State *env = cpu->env_ptr;
+    hwaddr kvmclock_struct_pa = env->system_time_msr & ~1ULL;
+    uint64_t migration_tsc = env->tsc;
+    struct pvclock_vcpu_time_info time;
+    uint64_t delta;
+    uint64_t nsec_lo;
+    uint64_t nsec_hi;
+    uint64_t nsec;
+
+    if (!(env->system_time_msr & 1ULL)) {
+        /* KVM clock not active */
+        return 0;
+    }
+
+    cpu_physical_memory_read(kvmclock_struct_pa, &time, sizeof(time));
+
+    assert(time.tsc_timestamp <= migration_tsc);
+    delta = migration_tsc - time.tsc_timestamp;
+    if (time.tsc_shift < 0) {
+        delta >>= -time.tsc_shift;
+    } else {
+        delta <<= time.tsc_shift;
+    }
+
+    mulu64(&nsec_lo, &nsec_hi, delta, time.tsc_to_system_mul);
+    nsec = (nsec_lo >> 32) | (nsec_hi << 32);
+    return nsec + time.system_time;
+}
 
 static void kvmclock_vm_state_change(void *opaque, int running,
                                      RunState state)
@@ -45,9 +89,15 @@
 
     if (running) {
         struct kvm_clock_data data;
+        uint64_t time_at_migration = kvmclock_current_nsec(s);
 
         s->clock_valid = false;
 
+	/* We can't rely on the migrated clock value, just discard it */
+	if (time_at_migration) {
+	        s->clock = time_at_migration;
+	}
+
         data.clock = s->clock;
         data.flags = 0;
         ret = kvm_vm_ioctl(kvm_state, KVM_SET_CLOCK, &data);
@@ -75,6 +125,8 @@
         if (s->clock_valid) {
             return;
         }
+
+        cpu_synchronize_all_states();
         ret = kvm_vm_ioctl(kvm_state, KVM_GET_CLOCK, &data);
         if (ret < 0) {
             fprintf(stderr, "KVM_GET_CLOCK failed: %s\n", strerror(ret));
diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c
index eaf3e61..a48e263 100644
--- a/hw/i386/pc_piix.c
+++ b/hw/i386/pc_piix.c
@@ -69,7 +69,7 @@
 static bool gigabyte_align = true;
 
 /* PC hardware initialisation */
-static void pc_init1(QEMUMachineInitArgs *args,
+static void pc_init1(MachineState *machine,
                      int pci_enabled,
                      int kvmclock_enabled)
 {
@@ -106,7 +106,7 @@
     object_property_add_child(qdev_get_machine(), "icc-bridge",
                               OBJECT(icc_bridge), NULL);
 
-    pc_cpus_init(args->cpu_model, icc_bridge);
+    pc_cpus_init(machine->cpu_model, icc_bridge);
 
     if (kvm_enabled() && kvmclock_enabled) {
         kvmclock_create();
@@ -119,13 +119,13 @@
      * For old machine types, use whatever split we used historically to avoid
      * breaking migration.
      */
-    if (args->ram_size >= 0xe0000000) {
+    if (machine->ram_size >= 0xe0000000) {
         ram_addr_t lowmem = gigabyte_align ? 0xc0000000 : 0xe0000000;
-        above_4g_mem_size = args->ram_size - lowmem;
+        above_4g_mem_size = machine->ram_size - lowmem;
         below_4g_mem_size = lowmem;
     } else {
         above_4g_mem_size = 0;
-        below_4g_mem_size = args->ram_size;
+        below_4g_mem_size = machine->ram_size;
     }
 
     if (pci_enabled) {
@@ -145,16 +145,17 @@
     guest_info->isapc_ram_fw = !pci_enabled;
 
     if (smbios_defaults) {
+        MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (i440FX + PIIX, 1996)",
-                            args->machine->name, smbios_legacy_mode);
+                            mc->name, smbios_legacy_mode);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
         fw_cfg = pc_memory_init(system_memory,
-                       args->kernel_filename, args->kernel_cmdline,
-                       args->initrd_filename,
+                       machine->kernel_filename, machine->kernel_cmdline,
+                       machine->initrd_filename,
                        below_4g_mem_size, above_4g_mem_size,
                        rom_memory, &ram_memory, guest_info);
     }
@@ -170,7 +171,7 @@
 
     if (pci_enabled) {
         pci_bus = i440fx_init(&i440fx_state, &piix3_devfn, &isa_bus, gsi,
-                              system_memory, system_io, args->ram_size,
+                              system_memory, system_io, machine->ram_size,
                               below_4g_mem_size,
                               above_4g_mem_size,
                               pci_memory, ram_memory);
@@ -235,7 +236,7 @@
         }
     }
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, args->boot_order,
+    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
                  floppy, idebus[0], idebus[1], rtc_state);
 
     if (pci_enabled && usb_enabled(false)) {
@@ -258,131 +259,131 @@
     }
 }
 
-static void pc_init_pci(QEMUMachineInitArgs *args)
+static void pc_init_pci(MachineState *machine)
 {
-    pc_init1(args, 1, 1);
+    pc_init1(machine, 1, 1);
 }
 
-static void pc_compat_2_0(QEMUMachineInitArgs *args)
+static void pc_compat_2_0(MachineState *machine)
 {
     smbios_legacy_mode = true;
 }
 
-static void pc_compat_1_7(QEMUMachineInitArgs *args)
+static void pc_compat_1_7(MachineState *machine)
 {
-    pc_compat_2_0(args);
+    pc_compat_2_0(machine);
     smbios_defaults = false;
     gigabyte_align = false;
     option_rom_has_mr = true;
     x86_cpu_compat_disable_kvm_features(FEAT_1_ECX, CPUID_EXT_X2APIC);
 }
 
-static void pc_compat_1_6(QEMUMachineInitArgs *args)
+static void pc_compat_1_6(MachineState *machine)
 {
-    pc_compat_1_7(args);
+    pc_compat_1_7(machine);
     has_pci_info = false;
     rom_file_has_mr = false;
     has_acpi_build = false;
 }
 
-static void pc_compat_1_5(QEMUMachineInitArgs *args)
+static void pc_compat_1_5(MachineState *machine)
 {
-    pc_compat_1_6(args);
+    pc_compat_1_6(machine);
 }
 
-static void pc_compat_1_4(QEMUMachineInitArgs *args)
+static void pc_compat_1_4(MachineState *machine)
 {
-    pc_compat_1_5(args);
+    pc_compat_1_5(machine);
     x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
     x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
-static void pc_compat_1_3(QEMUMachineInitArgs *args)
+static void pc_compat_1_3(MachineState *machine)
 {
-    pc_compat_1_4(args);
+    pc_compat_1_4(machine);
     enable_compat_apic_id_mode();
 }
 
 /* PC compat function for pc-0.14 to pc-1.2 */
-static void pc_compat_1_2(QEMUMachineInitArgs *args)
+static void pc_compat_1_2(MachineState *machine)
 {
-    pc_compat_1_3(args);
+    pc_compat_1_3(machine);
     x86_cpu_compat_disable_kvm_features(FEAT_KVM, KVM_FEATURE_PV_EOI);
 }
 
-static void pc_init_pci_2_0(QEMUMachineInitArgs *args)
+static void pc_init_pci_2_0(MachineState *machine)
 {
-    pc_compat_2_0(args);
-    pc_init_pci(args);
+    pc_compat_2_0(machine);
+    pc_init_pci(machine);
 }
 
-static void pc_init_pci_1_7(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_7(MachineState *machine)
 {
-    pc_compat_1_7(args);
-    pc_init_pci(args);
+    pc_compat_1_7(machine);
+    pc_init_pci(machine);
 }
 
-static void pc_init_pci_1_6(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_6(MachineState *machine)
 {
-    pc_compat_1_6(args);
-    pc_init_pci(args);
+    pc_compat_1_6(machine);
+    pc_init_pci(machine);
 }
 
-static void pc_init_pci_1_5(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_5(MachineState *machine)
 {
-    pc_compat_1_5(args);
-    pc_init_pci(args);
+    pc_compat_1_5(machine);
+    pc_init_pci(machine);
 }
 
-static void pc_init_pci_1_4(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_4(MachineState *machine)
 {
-    pc_compat_1_4(args);
-    pc_init_pci(args);
+    pc_compat_1_4(machine);
+    pc_init_pci(machine);
 }
 
-static void pc_init_pci_1_3(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_3(MachineState *machine)
 {
-    pc_compat_1_3(args);
-    pc_init_pci(args);
+    pc_compat_1_3(machine);
+    pc_init_pci(machine);
 }
 
 /* PC machine init function for pc-0.14 to pc-1.2 */
-static void pc_init_pci_1_2(QEMUMachineInitArgs *args)
+static void pc_init_pci_1_2(MachineState *machine)
 {
-    pc_compat_1_2(args);
-    pc_init_pci(args);
+    pc_compat_1_2(machine);
+    pc_init_pci(machine);
 }
 
 /* PC init function for pc-0.10 to pc-0.13, and reused by xenfv */
-static void pc_init_pci_no_kvmclock(QEMUMachineInitArgs *args)
+static void pc_init_pci_no_kvmclock(MachineState *machine)
 {
     has_pci_info = false;
     has_acpi_build = false;
     smbios_defaults = false;
     x86_cpu_compat_disable_kvm_features(FEAT_KVM, KVM_FEATURE_PV_EOI);
     enable_compat_apic_id_mode();
-    pc_init1(args, 1, 0);
+    pc_init1(machine, 1, 0);
 }
 
-static void pc_init_isa(QEMUMachineInitArgs *args)
+static void pc_init_isa(MachineState *machine)
 {
     has_pci_info = false;
     has_acpi_build = false;
     smbios_defaults = false;
-    if (!args->cpu_model) {
-        args->cpu_model = "486";
+    if (!machine->cpu_model) {
+        machine->cpu_model = "486";
     }
     x86_cpu_compat_disable_kvm_features(FEAT_KVM, KVM_FEATURE_PV_EOI);
     enable_compat_apic_id_mode();
-    pc_init1(args, 0, 1);
+    pc_init1(machine, 0, 1);
 }
 
 #ifdef CONFIG_XEN
-static void pc_xen_hvm_init(QEMUMachineInitArgs *args)
+static void pc_xen_hvm_init(MachineState *machine)
 {
     PCIBus *bus;
 
-    pc_init_pci(args);
+    pc_init_pci(machine);
 
     bus = pci_find_primary_bus();
     if (bus != NULL) {
diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c
index 9517ec6..b3c02c1 100644
--- a/hw/i386/pc_q35.c
+++ b/hw/i386/pc_q35.c
@@ -59,7 +59,7 @@
 static bool gigabyte_align = true;
 
 /* PC hardware initialisation */
-static void pc_q35_init(QEMUMachineInitArgs *args)
+static void pc_q35_init(MachineState *machine)
 {
     ram_addr_t below_4g_mem_size, above_4g_mem_size;
     Q35PCIHost *q35_host;
@@ -93,7 +93,7 @@
     object_property_add_child(qdev_get_machine(), "icc-bridge",
                               OBJECT(icc_bridge), NULL);
 
-    pc_cpus_init(args->cpu_model, icc_bridge);
+    pc_cpus_init(machine->cpu_model, icc_bridge);
     pc_acpi_init("q35-acpi-dsdt.aml");
 
     kvmclock_create();
@@ -107,13 +107,13 @@
      * For old machine types, use whatever split we used historically to avoid
      * breaking migration.
      */
-    if (args->ram_size >= 0xb0000000) {
+    if (machine->ram_size >= 0xb0000000) {
         ram_addr_t lowmem = gigabyte_align ? 0x80000000 : 0xb0000000;
-        above_4g_mem_size = args->ram_size - lowmem;
+        above_4g_mem_size = machine->ram_size - lowmem;
         below_4g_mem_size = lowmem;
     } else {
         above_4g_mem_size = 0;
-        below_4g_mem_size = args->ram_size;
+        below_4g_mem_size = machine->ram_size;
     }
 
     /* pci enabled */
@@ -132,16 +132,17 @@
     guest_info->has_acpi_build = has_acpi_build;
 
     if (smbios_defaults) {
+        MachineClass *mc = MACHINE_GET_CLASS(machine);
         /* These values are guest ABI, do not change */
         smbios_set_defaults("QEMU", "Standard PC (Q35 + ICH9, 2009)",
-                            args->machine->name, smbios_legacy_mode);
+                            mc->name, smbios_legacy_mode);
     }
 
     /* allocate ram and load rom/bios */
     if (!xen_enabled()) {
         pc_memory_init(get_system_memory(),
-                       args->kernel_filename, args->kernel_cmdline,
-                       args->initrd_filename,
+                       machine->kernel_filename, machine->kernel_cmdline,
+                       machine->initrd_filename,
                        below_4g_mem_size, above_4g_mem_size,
                        rom_memory, &ram_memory, guest_info);
     }
@@ -230,7 +231,7 @@
                                     0xb100),
                       8, NULL, 0);
 
-    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, args->boot_order,
+    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, machine->boot_order,
                  floppy, idebus[0], idebus[1], rtc_state);
 
     /* the rest devices to which pci devfn is automatically assigned */
@@ -241,68 +242,68 @@
     }
 }
 
-static void pc_compat_2_0(QEMUMachineInitArgs *args)
+static void pc_compat_2_0(MachineState *machine)
 {
     smbios_legacy_mode = true;
 }
 
-static void pc_compat_1_7(QEMUMachineInitArgs *args)
+static void pc_compat_1_7(MachineState *machine)
 {
-    pc_compat_2_0(args);
+    pc_compat_2_0(machine);
     smbios_defaults = false;
     gigabyte_align = false;
     option_rom_has_mr = true;
     x86_cpu_compat_disable_kvm_features(FEAT_1_ECX, CPUID_EXT_X2APIC);
 }
 
-static void pc_compat_1_6(QEMUMachineInitArgs *args)
+static void pc_compat_1_6(MachineState *machine)
 {
-    pc_compat_1_7(args);
+    pc_compat_1_7(machine);
     has_pci_info = false;
     rom_file_has_mr = false;
     has_acpi_build = false;
 }
 
-static void pc_compat_1_5(QEMUMachineInitArgs *args)
+static void pc_compat_1_5(MachineState *machine)
 {
-    pc_compat_1_6(args);
+    pc_compat_1_6(machine);
 }
 
-static void pc_compat_1_4(QEMUMachineInitArgs *args)
+static void pc_compat_1_4(MachineState *machine)
 {
-    pc_compat_1_5(args);
+    pc_compat_1_5(machine);
     x86_cpu_compat_set_features("n270", FEAT_1_ECX, 0, CPUID_EXT_MOVBE);
     x86_cpu_compat_set_features("Westmere", FEAT_1_ECX, 0, CPUID_EXT_PCLMULQDQ);
 }
 
-static void pc_q35_init_2_0(QEMUMachineInitArgs *args)
+static void pc_q35_init_2_0(MachineState *machine)
 {
-    pc_compat_2_0(args);
-    pc_q35_init(args);
+    pc_compat_2_0(machine);
+    pc_q35_init(machine);
 }
 
-static void pc_q35_init_1_7(QEMUMachineInitArgs *args)
+static void pc_q35_init_1_7(MachineState *machine)
 {
-    pc_compat_1_7(args);
-    pc_q35_init(args);
+    pc_compat_1_7(machine);
+    pc_q35_init(machine);
 }
 
-static void pc_q35_init_1_6(QEMUMachineInitArgs *args)
+static void pc_q35_init_1_6(MachineState *machine)
 {
-    pc_compat_1_6(args);
-    pc_q35_init(args);
+    pc_compat_1_6(machine);
+    pc_q35_init(machine);
 }
 
-static void pc_q35_init_1_5(QEMUMachineInitArgs *args)
+static void pc_q35_init_1_5(MachineState *machine)
 {
-    pc_compat_1_5(args);
-    pc_q35_init(args);
+    pc_compat_1_5(machine);
+    pc_q35_init(machine);
 }
 
-static void pc_q35_init_1_4(QEMUMachineInitArgs *args)
+static void pc_q35_init_1_4(MachineState *machine)
 {
-    pc_compat_1_4(args);
-    pc_q35_init(args);
+    pc_compat_1_4(machine);
+    pc_q35_init(machine);
 }
 
 #define PC_Q35_MACHINE_OPTIONS \
diff --git a/hw/input/hid.c b/hw/input/hid.c
index bb0fa6a..295bdab 100644
--- a/hw/input/hid.c
+++ b/hw/input/hid.c
@@ -105,70 +105,135 @@
     }
 }
 
-static void hid_pointer_event_clear(HIDPointerEvent *e, int buttons)
+static void hid_pointer_event(DeviceState *dev, QemuConsole *src,
+                              InputEvent *evt)
 {
-    e->xdx = e->ydy = e->dz = 0;
-    e->buttons_state = buttons;
+    static const int bmap[INPUT_BUTTON_MAX] = {
+        [INPUT_BUTTON_LEFT]   = 0x01,
+        [INPUT_BUTTON_RIGHT]  = 0x02,
+        [INPUT_BUTTON_MIDDLE] = 0x04,
+    };
+    HIDState *hs = (HIDState *)dev;
+    HIDPointerEvent *e;
+
+    assert(hs->n < QUEUE_LENGTH);
+    e = &hs->ptr.queue[(hs->head + hs->n) & QUEUE_MASK];
+
+    switch (evt->kind) {
+    case INPUT_EVENT_KIND_REL:
+        if (evt->rel->axis == INPUT_AXIS_X) {
+            e->xdx += evt->rel->value;
+        } else if (evt->rel->axis == INPUT_AXIS_Y) {
+            e->ydy -= evt->rel->value;
+        }
+        break;
+
+    case INPUT_EVENT_KIND_ABS:
+        if (evt->rel->axis == INPUT_AXIS_X) {
+            e->xdx = evt->rel->value;
+        } else if (evt->rel->axis == INPUT_AXIS_Y) {
+            e->ydy = evt->rel->value;
+        }
+        break;
+
+    case INPUT_EVENT_KIND_BTN:
+        if (evt->btn->down) {
+            e->buttons_state |= bmap[evt->btn->button];
+            if (evt->btn->button == INPUT_BUTTON_WHEEL_UP) {
+                e->dz--;
+            } else if (evt->btn->button == INPUT_BUTTON_WHEEL_DOWN) {
+                e->dz++;
+            }
+        } else {
+            e->buttons_state &= ~bmap[evt->btn->button];
+        }
+        break;
+
+    default:
+        /* keep gcc happy */
+        break;
+    }
+
 }
 
-static void hid_pointer_event_combine(HIDPointerEvent *e, int xyrel,
-                                      int x1, int y1, int z1) {
-    if (xyrel) {
-        e->xdx += x1;
-        e->ydy += y1;
-    } else {
-        e->xdx = x1;
-        e->ydy = y1;
-        /* Windows drivers do not like the 0/0 position and ignore such
-         * events. */
-        if (!(x1 | y1)) {
-            e->xdx = 1;
+static void hid_pointer_sync(DeviceState *dev)
+{
+    HIDState *hs = (HIDState *)dev;
+    HIDPointerEvent *prev, *curr, *next;
+    bool event_compression = false;
+
+    if (hs->n == QUEUE_LENGTH-1) {
+        /*
+         * Queue full.  We are loosing information, but we at least
+         * keep track of most recent button state.
+         */
+        return;
+    }
+
+    prev = &hs->ptr.queue[(hs->head + hs->n - 1) & QUEUE_MASK];
+    curr = &hs->ptr.queue[(hs->head + hs->n) & QUEUE_MASK];
+    next = &hs->ptr.queue[(hs->head + hs->n + 1) & QUEUE_MASK];
+
+    if (hs->n > 0) {
+        /*
+         * No button state change between previous and current event
+         * (and previous wasn't seen by the guest yet), so there is
+         * motion information only and we can combine the two event
+         * into one.
+         */
+        if (curr->buttons_state == prev->buttons_state) {
+            event_compression = true;
         }
     }
-    e->dz += z1;
-}
 
-static void hid_pointer_event(void *opaque,
-                              int x1, int y1, int z1, int buttons_state)
-{
-    HIDState *hs = opaque;
-    unsigned use_slot = (hs->head + hs->n - 1) & QUEUE_MASK;
-    unsigned previous_slot = (use_slot - 1) & QUEUE_MASK;
-
-    /* We combine events where feasible to keep the queue small.  We shouldn't
-     * combine anything with the first event of a particular button state, as
-     * that would change the location of the button state change.  When the
-     * queue is empty, a second event is needed because we don't know if
-     * the first event changed the button state.  */
-    if (hs->n == QUEUE_LENGTH) {
-        /* Queue full.  Discard old button state, combine motion normally.  */
-        hs->ptr.queue[use_slot].buttons_state = buttons_state;
-    } else if (hs->n < 2 ||
-               hs->ptr.queue[use_slot].buttons_state != buttons_state ||
-               hs->ptr.queue[previous_slot].buttons_state !=
-               hs->ptr.queue[use_slot].buttons_state) {
-        /* Cannot or should not combine, so add an empty item to the queue.  */
-        QUEUE_INCR(use_slot);
+    if (event_compression) {
+        /* add current motion to previous, clear current */
+        if (hs->kind == HID_MOUSE) {
+            prev->xdx += curr->xdx;
+            curr->xdx = 0;
+            prev->ydy -= curr->ydy;
+            curr->ydy = 0;
+        } else {
+            prev->xdx = curr->xdx;
+            prev->ydy = curr->ydy;
+        }
+        prev->dz += curr->dz;
+        curr->dz = 0;
+    } else {
+        /* prepate next (clear rel, copy abs + btns) */
+        if (hs->kind == HID_MOUSE) {
+            next->xdx = 0;
+            next->ydy = 0;
+        } else {
+            next->xdx = curr->xdx;
+            next->ydy = curr->ydy;
+        }
+        next->dz = 0;
+        next->buttons_state = curr->buttons_state;
+        /* make current guest visible, notify guest */
         hs->n++;
-        hid_pointer_event_clear(&hs->ptr.queue[use_slot], buttons_state);
+        hs->event(hs);
     }
-    hid_pointer_event_combine(&hs->ptr.queue[use_slot],
-                              hs->kind == HID_MOUSE,
-                              x1, y1, z1);
-    hs->event(hs);
 }
 
-static void hid_keyboard_event(void *opaque, int keycode)
+static void hid_keyboard_event(DeviceState *dev, QemuConsole *src,
+                               InputEvent *evt)
 {
-    HIDState *hs = opaque;
+    HIDState *hs = (HIDState *)dev;
+    int scancodes[3], i, count;
     int slot;
 
-    if (hs->n == QUEUE_LENGTH) {
+    count = qemu_input_key_value_to_scancode(evt->key->key,
+                                             evt->key->down,
+                                             scancodes);
+    if (hs->n + count > QUEUE_LENGTH) {
         fprintf(stderr, "usb-kbd: warning: key event queue full\n");
         return;
     }
-    slot = (hs->head + hs->n) & QUEUE_MASK; hs->n++;
-    hs->kbd.keycodes[slot] = keycode;
+    for (i = 0; i < count; i++) {
+        slot = (hs->head + hs->n) & QUEUE_MASK; hs->n++;
+        hs->kbd.keycodes[slot] = scancodes[i];
+    }
     hs->event(hs);
 }
 
@@ -247,14 +312,14 @@
 void hid_pointer_activate(HIDState *hs)
 {
     if (!hs->ptr.mouse_grabbed) {
-        qemu_activate_mouse_event_handler(hs->ptr.eh_entry);
+        qemu_input_handler_activate(hs->s);
         hs->ptr.mouse_grabbed = 1;
     }
 }
 
 int hid_pointer_poll(HIDState *hs, uint8_t *buf, int len)
 {
-    int dx, dy, dz, b, l;
+    int dx, dy, dz, l;
     int index;
     HIDPointerEvent *e;
 
@@ -279,17 +344,6 @@
     dz = int_clamp(e->dz, -127, 127);
     e->dz -= dz;
 
-    b = 0;
-    if (e->buttons_state & MOUSE_EVENT_LBUTTON) {
-        b |= 0x01;
-    }
-    if (e->buttons_state & MOUSE_EVENT_RBUTTON) {
-        b |= 0x02;
-    }
-    if (e->buttons_state & MOUSE_EVENT_MBUTTON) {
-        b |= 0x04;
-    }
-
     if (hs->n &&
         !e->dz &&
         (hs->kind == HID_TABLET || (!e->xdx && !e->ydy))) {
@@ -304,7 +358,7 @@
     switch (hs->kind) {
     case HID_MOUSE:
         if (len > l) {
-            buf[l++] = b;
+            buf[l++] = e->buttons_state;
         }
         if (len > l) {
             buf[l++] = dx;
@@ -319,7 +373,7 @@
 
     case HID_TABLET:
         if (len > l) {
-            buf[l++] = b;
+            buf[l++] = e->buttons_state;
         }
         if (len > l) {
             buf[l++] = dx & 0xff;
@@ -413,31 +467,45 @@
 
 void hid_free(HIDState *hs)
 {
-    switch (hs->kind) {
-    case HID_KEYBOARD:
-        qemu_remove_kbd_event_handler(hs->kbd.eh_entry);
-        break;
-    case HID_MOUSE:
-    case HID_TABLET:
-        qemu_remove_mouse_event_handler(hs->ptr.eh_entry);
-        break;
-    }
+    qemu_input_handler_unregister(hs->s);
     hid_del_idle_timer(hs);
 }
 
+static QemuInputHandler hid_keyboard_handler = {
+    .name  = "QEMU HID Keyboard",
+    .mask  = INPUT_EVENT_MASK_KEY,
+    .event = hid_keyboard_event,
+};
+
+static QemuInputHandler hid_mouse_handler = {
+    .name  = "QEMU HID Mouse",
+    .mask  = INPUT_EVENT_MASK_BTN | INPUT_EVENT_MASK_REL,
+    .event = hid_pointer_event,
+    .sync  = hid_pointer_sync,
+};
+
+static QemuInputHandler hid_tablet_handler = {
+    .name  = "QEMU HID Tablet",
+    .mask  = INPUT_EVENT_MASK_BTN | INPUT_EVENT_MASK_ABS,
+    .event = hid_pointer_event,
+    .sync  = hid_pointer_sync,
+};
+
 void hid_init(HIDState *hs, int kind, HIDEventFunc event)
 {
     hs->kind = kind;
     hs->event = event;
 
     if (hs->kind == HID_KEYBOARD) {
-        hs->kbd.eh_entry = qemu_add_kbd_event_handler(hid_keyboard_event, hs);
+        hs->s = qemu_input_handler_register((DeviceState *)hs,
+                                            &hid_keyboard_handler);
+        qemu_input_handler_activate(hs->s);
     } else if (hs->kind == HID_MOUSE) {
-        hs->ptr.eh_entry = qemu_add_mouse_event_handler(hid_pointer_event, hs,
-                                                        0, "QEMU HID Mouse");
+        hs->s = qemu_input_handler_register((DeviceState *)hs,
+                                            &hid_mouse_handler);
     } else if (hs->kind == HID_TABLET) {
-        hs->ptr.eh_entry = qemu_add_mouse_event_handler(hid_pointer_event, hs,
-                                                        1, "QEMU HID Tablet");
+        hs->s = qemu_input_handler_register((DeviceState *)hs,
+                                            &hid_tablet_handler);
     }
 }
 
diff --git a/hw/intc/Makefile.objs b/hw/intc/Makefile.objs
index c8a2318..843864a 100644
--- a/hw/intc/Makefile.objs
+++ b/hw/intc/Makefile.objs
@@ -26,3 +26,4 @@
 obj-$(CONFIG_XICS_KVM) += xics_kvm.o
 obj-$(CONFIG_ALLWINNER_A10_PIC) += allwinner-a10-pic.o
 obj-$(CONFIG_S390_FLIC) += s390_flic.o
+obj-$(CONFIG_S390_FLIC_KVM) += s390_flic_kvm.o
diff --git a/hw/intc/s390_flic.c b/hw/intc/s390_flic.c
index b2ef3e3..03c5e89 100644
--- a/hw/intc/s390_flic.c
+++ b/hw/intc/s390_flic.c
@@ -1,322 +1,103 @@
 /*
- * QEMU S390x KVM floating interrupt controller (flic)
+ * QEMU S390x floating interrupt controller (flic)
  *
  * Copyright 2014 IBM Corp.
  * Author(s): Jens Freimann <jfrei@linux.vnet.ibm.com>
+ *            Cornelia Huck <cornelia.huck@de.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at
  * your option) any later version. See the COPYING file in the top-level
  * directory.
  */
 
-#include <sys/ioctl.h>
 #include "qemu/error-report.h"
 #include "hw/sysbus.h"
-#include "sysemu/kvm.h"
 #include "migration/qemu-file.h"
 #include "hw/s390x/s390_flic.h"
 #include "trace.h"
 
-#define FLIC_SAVE_INITIAL_SIZE getpagesize()
-#define FLIC_FAILED (-1UL)
-#define FLIC_SAVEVM_VERSION 1
+S390FLICState *s390_get_flic(void)
+{
+    S390FLICState *fs;
+
+    fs = S390_FLIC_COMMON(object_resolve_path(TYPE_KVM_S390_FLIC, NULL));
+    if (!fs) {
+        fs = S390_FLIC_COMMON(object_resolve_path(TYPE_QEMU_S390_FLIC, NULL));
+    }
+    return fs;
+}
 
 void s390_flic_init(void)
 {
     DeviceState *dev;
     int r;
 
-    if (kvm_enabled()) {
-        dev = qdev_create(NULL, "s390-flic");
-        object_property_add_child(qdev_get_machine(), "s390-flic",
-                                OBJECT(dev), NULL);
-        r = qdev_init(dev);
-        if (r) {
-            error_report("flic: couldn't create qdev");
-        }
+    dev = s390_flic_kvm_create();
+    if (!dev) {
+        dev = qdev_create(NULL, TYPE_QEMU_S390_FLIC);
+        object_property_add_child(qdev_get_machine(), TYPE_QEMU_S390_FLIC,
+                                  OBJECT(dev), NULL);
+    }
+    r = qdev_init(dev);
+    if (r) {
+        error_report("flic: couldn't create qdev");
     }
 }
 
-/**
- * flic_get_all_irqs - store all pending irqs in buffer
- * @buf: pointer to buffer which is passed to kernel
- * @len: length of buffer
- * @flic: pointer to flic device state
- *
- * Returns: -ENOMEM if buffer is too small,
- * -EINVAL if attr.group is invalid,
- * -EFAULT if copying to userspace failed,
- * on success return number of stored interrupts
- */
-static int flic_get_all_irqs(KVMS390FLICState *flic,
-                             void *buf, int len)
+static int qemu_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
+                                         uint8_t isc, bool swap,
+                                         bool is_maskable)
 {
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_FLIC_GET_ALL_IRQS,
-        .addr = (uint64_t) buf,
-        .attr = len,
-    };
-    int rc;
-
-    rc = ioctl(flic->fd, KVM_GET_DEVICE_ATTR, &attr);
-
-    return rc == -1 ? -errno : rc;
+    /* nothing to do */
+    return 0;
 }
 
-static void flic_enable_pfault(KVMS390FLICState *flic)
+static int qemu_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
+                                    uint64_t map_addr, bool do_map)
 {
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_FLIC_APF_ENABLE,
-    };
-    int rc;
-
-    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
-
-    if (rc) {
-        fprintf(stderr, "flic: couldn't enable pfault\n");
-    }
+    /* nothing to do */
+    return 0;
 }
 
-static void flic_disable_wait_pfault(KVMS390FLICState *flic)
+static int qemu_s390_add_adapter_routes(S390FLICState *fs,
+                                        AdapterRoutes *routes)
 {
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_FLIC_APF_DISABLE_WAIT,
-    };
-    int rc;
-
-    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
-
-    if (rc) {
-        fprintf(stderr, "flic: couldn't disable pfault\n");
-    }
+    return -ENOSYS;
 }
 
-/** flic_enqueue_irqs - returns 0 on success
- * @buf: pointer to buffer which is passed to kernel
- * @len: length of buffer
- * @flic: pointer to flic device state
- *
- * Returns: -EINVAL if attr.group is unknown
- */
-static int flic_enqueue_irqs(void *buf, uint64_t len,
-                            KVMS390FLICState *flic)
+static void qemu_s390_release_adapter_routes(S390FLICState *fs,
+                                             AdapterRoutes *routes)
 {
-    int rc;
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_FLIC_ENQUEUE,
-        .addr = (uint64_t) buf,
-        .attr = len,
-    };
-
-    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
-
-    return rc ? -errno : 0;
 }
 
-/**
- * __get_all_irqs - store all pending irqs in buffer
- * @flic: pointer to flic device state
- * @buf: pointer to pointer to a buffer
- * @len: length of buffer
- *
- * Returns: return value of flic_get_all_irqs
- * Note: Retry and increase buffer size until flic_get_all_irqs
- * either returns a value >= 0 or a negative error code.
- * -ENOMEM is an exception, which means the buffer is too small
- * and we should try again. Other negative error codes can be
- * -EFAULT and -EINVAL which we ignore at this point
- */
-static int __get_all_irqs(KVMS390FLICState *flic,
-                          void **buf, int len)
+static void qemu_s390_flic_class_init(ObjectClass *oc, void *data)
 {
-    int r;
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
 
-    do {
-        /* returns -ENOMEM if buffer is too small and number
-         * of queued interrupts on success */
-        r = flic_get_all_irqs(flic, *buf, len);
-        if (r >= 0) {
-            break;
-        }
-        len *= 2;
-        *buf = g_try_realloc(*buf, len);
-        if (!buf) {
-            return -ENOMEM;
-        }
-    } while (r == -ENOMEM && len <= KVM_S390_FLIC_MAX_BUFFER);
-
-    return r;
+    fsc->register_io_adapter = qemu_s390_register_io_adapter;
+    fsc->io_adapter_map = qemu_s390_io_adapter_map;
+    fsc->add_adapter_routes = qemu_s390_add_adapter_routes;
+    fsc->release_adapter_routes = qemu_s390_release_adapter_routes;
 }
 
-/**
- * kvm_flic_save - Save pending floating interrupts
- * @f: QEMUFile containing migration state
- * @opaque: pointer to flic device state
- *
- * Note: Pass buf and len to kernel. Start with one page and
- * increase until buffer is sufficient or maxium size is
- * reached
- */
-static void kvm_flic_save(QEMUFile *f, void *opaque)
-{
-    KVMS390FLICState *flic = opaque;
-    int len = FLIC_SAVE_INITIAL_SIZE;
-    void *buf;
-    int count;
-
-    flic_disable_wait_pfault((struct KVMS390FLICState *) opaque);
-
-    buf = g_try_malloc0(len);
-    if (!buf) {
-        /* Storing FLIC_FAILED into the count field here will cause the
-         * target system to fail when attempting to load irqs from the
-         * migration state */
-        error_report("flic: couldn't allocate memory");
-        qemu_put_be64(f, FLIC_FAILED);
-        return;
-    }
-
-    count = __get_all_irqs(flic, &buf, len);
-    if (count < 0) {
-        error_report("flic: couldn't retrieve irqs from kernel, rc %d",
-                     count);
-        /* Storing FLIC_FAILED into the count field here will cause the
-         * target system to fail when attempting to load irqs from the
-         * migration state */
-        qemu_put_be64(f, FLIC_FAILED);
-    } else {
-        qemu_put_be64(f, count);
-        qemu_put_buffer(f, (uint8_t *) buf,
-                        count * sizeof(struct kvm_s390_irq));
-    }
-    g_free(buf);
-}
-
-/**
- * kvm_flic_load - Load pending floating interrupts
- * @f: QEMUFile containing migration state
- * @opaque: pointer to flic device state
- * @version_id: version id for migration
- *
- * Returns: value of flic_enqueue_irqs, -EINVAL on error
- * Note: Do nothing when no interrupts where stored
- * in QEMUFile
- */
-static int kvm_flic_load(QEMUFile *f, void *opaque, int version_id)
-{
-    uint64_t len = 0;
-    uint64_t count = 0;
-    void *buf = NULL;
-    int r = 0;
-
-    if (version_id != FLIC_SAVEVM_VERSION) {
-        r = -EINVAL;
-        goto out;
-    }
-
-    flic_enable_pfault((struct KVMS390FLICState *) opaque);
-
-    count = qemu_get_be64(f);
-    len = count * sizeof(struct kvm_s390_irq);
-    if (count == FLIC_FAILED) {
-        r = -EINVAL;
-        goto out;
-    }
-    if (count == 0) {
-        r = 0;
-        goto out;
-    }
-    buf = g_try_malloc0(len);
-    if (!buf) {
-        r = -ENOMEM;
-        goto out;
-    }
-
-    if (qemu_get_buffer(f, (uint8_t *) buf, len) != len) {
-        r = -EINVAL;
-        goto out_free;
-    }
-    r = flic_enqueue_irqs(buf, len, (struct KVMS390FLICState *) opaque);
-
-out_free:
-    g_free(buf);
-out:
-    return r;
-}
-
-static void kvm_s390_flic_realize(DeviceState *dev, Error **errp)
-{
-    KVMS390FLICState *flic_state = KVM_S390_FLIC(dev);
-    struct kvm_create_device cd = {0};
-    int ret;
-
-    flic_state->fd = -1;
-    if (!kvm_check_extension(kvm_state, KVM_CAP_DEVICE_CTRL)) {
-        trace_flic_no_device_api(errno);
-        return;
-    }
-
-    cd.type = KVM_DEV_TYPE_FLIC;
-    ret = kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd);
-    if (ret < 0) {
-        trace_flic_create_device(errno);
-        return;
-    }
-    flic_state->fd = cd.fd;
-
-    /* Register savevm handler for floating interrupts */
-    register_savevm(NULL, "s390-flic", 0, 1, kvm_flic_save,
-                    kvm_flic_load, (void *) flic_state);
-}
-
-static void kvm_s390_flic_unrealize(DeviceState *dev, Error **errp)
-{
-    KVMS390FLICState *flic_state = KVM_S390_FLIC(dev);
-
-    unregister_savevm(DEVICE(flic_state), "s390-flic", flic_state);
-}
-
-static void kvm_s390_flic_reset(DeviceState *dev)
-{
-    KVMS390FLICState *flic = KVM_S390_FLIC(dev);
-    struct kvm_device_attr attr = {
-        .group = KVM_DEV_FLIC_CLEAR_IRQS,
-    };
-    int rc = 0;
-
-    if (flic->fd == -1) {
-        return;
-    }
-
-    flic_disable_wait_pfault(flic);
-
-    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
-    if (rc) {
-        trace_flic_reset_failed(errno);
-    }
-
-    flic_enable_pfault(flic);
-}
-
-static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(oc);
-
-    dc->realize = kvm_s390_flic_realize;
-    dc->unrealize = kvm_s390_flic_unrealize;
-    dc->reset = kvm_s390_flic_reset;
-}
-
-static const TypeInfo kvm_s390_flic_info = {
-    .name          = TYPE_KVM_S390_FLIC,
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(KVMS390FLICState),
-    .class_init    = kvm_s390_flic_class_init,
+static const TypeInfo qemu_s390_flic_info = {
+    .name          = TYPE_QEMU_S390_FLIC,
+    .parent        = TYPE_S390_FLIC_COMMON,
+    .instance_size = sizeof(QEMUS390FLICState),
+    .class_init    = qemu_s390_flic_class_init,
 };
 
-static void kvm_s390_flic_register_types(void)
+static const TypeInfo s390_flic_common_info = {
+    .name          = TYPE_S390_FLIC_COMMON,
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(S390FLICState),
+    .class_size    = sizeof(S390FLICStateClass),
+};
+
+static void qemu_s390_flic_register_types(void)
 {
-    type_register_static(&kvm_s390_flic_info);
+    type_register_static(&s390_flic_common_info);
+    type_register_static(&qemu_s390_flic_info);
 }
 
-type_init(kvm_s390_flic_register_types)
+type_init(qemu_s390_flic_register_types)
diff --git a/hw/intc/s390_flic_kvm.c b/hw/intc/s390_flic_kvm.c
new file mode 100644
index 0000000..46c9e61
--- /dev/null
+++ b/hw/intc/s390_flic_kvm.c
@@ -0,0 +1,420 @@
+/*
+ * QEMU S390x KVM floating interrupt controller (flic)
+ *
+ * Copyright 2014 IBM Corp.
+ * Author(s): Jens Freimann <jfrei@linux.vnet.ibm.com>
+ *            Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <sys/ioctl.h>
+#include "qemu/error-report.h"
+#include "hw/sysbus.h"
+#include "sysemu/kvm.h"
+#include "migration/qemu-file.h"
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/adapter.h"
+#include "trace.h"
+
+#define FLIC_SAVE_INITIAL_SIZE getpagesize()
+#define FLIC_FAILED (-1UL)
+#define FLIC_SAVEVM_VERSION 1
+
+typedef struct KVMS390FLICState {
+    S390FLICState parent_obj;
+
+    uint32_t fd;
+} KVMS390FLICState;
+
+DeviceState *s390_flic_kvm_create(void)
+{
+    DeviceState *dev = NULL;
+
+    if (kvm_enabled()) {
+        dev = qdev_create(NULL, TYPE_KVM_S390_FLIC);
+        object_property_add_child(qdev_get_machine(), TYPE_KVM_S390_FLIC,
+                                  OBJECT(dev), NULL);
+    }
+    return dev;
+}
+
+/**
+ * flic_get_all_irqs - store all pending irqs in buffer
+ * @buf: pointer to buffer which is passed to kernel
+ * @len: length of buffer
+ * @flic: pointer to flic device state
+ *
+ * Returns: -ENOMEM if buffer is too small,
+ * -EINVAL if attr.group is invalid,
+ * -EFAULT if copying to userspace failed,
+ * on success return number of stored interrupts
+ */
+static int flic_get_all_irqs(KVMS390FLICState *flic,
+                             void *buf, int len)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_GET_ALL_IRQS,
+        .addr = (uint64_t) buf,
+        .attr = len,
+    };
+    int rc;
+
+    rc = ioctl(flic->fd, KVM_GET_DEVICE_ATTR, &attr);
+
+    return rc == -1 ? -errno : rc;
+}
+
+static void flic_enable_pfault(KVMS390FLICState *flic)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_APF_ENABLE,
+    };
+    int rc;
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    if (rc) {
+        fprintf(stderr, "flic: couldn't enable pfault\n");
+    }
+}
+
+static void flic_disable_wait_pfault(KVMS390FLICState *flic)
+{
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_APF_DISABLE_WAIT,
+    };
+    int rc;
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    if (rc) {
+        fprintf(stderr, "flic: couldn't disable pfault\n");
+    }
+}
+
+/** flic_enqueue_irqs - returns 0 on success
+ * @buf: pointer to buffer which is passed to kernel
+ * @len: length of buffer
+ * @flic: pointer to flic device state
+ *
+ * Returns: -EINVAL if attr.group is unknown
+ */
+static int flic_enqueue_irqs(void *buf, uint64_t len,
+                            KVMS390FLICState *flic)
+{
+    int rc;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_ENQUEUE,
+        .addr = (uint64_t) buf,
+        .attr = len,
+    };
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    return rc ? -errno : 0;
+}
+
+/**
+ * __get_all_irqs - store all pending irqs in buffer
+ * @flic: pointer to flic device state
+ * @buf: pointer to pointer to a buffer
+ * @len: length of buffer
+ *
+ * Returns: return value of flic_get_all_irqs
+ * Note: Retry and increase buffer size until flic_get_all_irqs
+ * either returns a value >= 0 or a negative error code.
+ * -ENOMEM is an exception, which means the buffer is too small
+ * and we should try again. Other negative error codes can be
+ * -EFAULT and -EINVAL which we ignore at this point
+ */
+static int __get_all_irqs(KVMS390FLICState *flic,
+                          void **buf, int len)
+{
+    int r;
+
+    do {
+        /* returns -ENOMEM if buffer is too small and number
+         * of queued interrupts on success */
+        r = flic_get_all_irqs(flic, *buf, len);
+        if (r >= 0) {
+            break;
+        }
+        len *= 2;
+        *buf = g_try_realloc(*buf, len);
+        if (!buf) {
+            return -ENOMEM;
+        }
+    } while (r == -ENOMEM && len <= KVM_S390_FLIC_MAX_BUFFER);
+
+    return r;
+}
+
+static int kvm_s390_register_io_adapter(S390FLICState *fs, uint32_t id,
+                                        uint8_t isc, bool swap,
+                                        bool is_maskable)
+{
+    struct kvm_s390_io_adapter adapter = {
+        .id = id,
+        .isc = isc,
+        .maskable = is_maskable,
+        .swap = swap,
+    };
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    int r, ret;
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_ADAPTER_REGISTER,
+        .addr = (uint64_t)&adapter,
+    };
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
+        return -ENOSYS;
+    }
+
+    r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+
+    ret = r ? -errno : 0;
+    return ret;
+}
+
+static int kvm_s390_io_adapter_map(S390FLICState *fs, uint32_t id,
+                                   uint64_t map_addr, bool do_map)
+{
+    struct kvm_s390_io_adapter_req req = {
+        .id = id,
+        .type = do_map ? KVM_S390_IO_ADAPTER_MAP : KVM_S390_IO_ADAPTER_UNMAP,
+        .addr = map_addr,
+    };
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_ADAPTER_MODIFY,
+        .addr = (uint64_t)&req,
+    };
+    KVMS390FLICState *flic = KVM_S390_FLIC(fs);
+    int r;
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_IRQ_ROUTING)) {
+        return -ENOSYS;
+    }
+
+    r = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+    return r ? -errno : 0;
+}
+
+static int kvm_s390_add_adapter_routes(S390FLICState *fs,
+                                       AdapterRoutes *routes)
+{
+    int ret, i;
+    uint64_t ind_offset = routes->adapter.ind_offset;
+
+    for (i = 0; i < routes->num_routes; i++) {
+        ret = kvm_irqchip_add_adapter_route(kvm_state, &routes->adapter);
+        if (ret < 0) {
+            goto out_undo;
+        }
+        routes->gsi[i] = ret;
+        routes->adapter.ind_offset++;
+    }
+    /* Restore passed-in structure to original state. */
+    routes->adapter.ind_offset = ind_offset;
+    return 0;
+out_undo:
+    while (--i >= 0) {
+        kvm_irqchip_release_virq(kvm_state, routes->gsi[i]);
+        routes->gsi[i] = -1;
+    }
+    routes->adapter.ind_offset = ind_offset;
+    return ret;
+}
+
+static void kvm_s390_release_adapter_routes(S390FLICState *fs,
+                                            AdapterRoutes *routes)
+{
+    int i;
+
+    for (i = 0; i < routes->num_routes; i++) {
+        if (routes->gsi[i] >= 0) {
+            kvm_irqchip_release_virq(kvm_state, routes->gsi[i]);
+            routes->gsi[i] = -1;
+        }
+    }
+}
+
+/**
+ * kvm_flic_save - Save pending floating interrupts
+ * @f: QEMUFile containing migration state
+ * @opaque: pointer to flic device state
+ *
+ * Note: Pass buf and len to kernel. Start with one page and
+ * increase until buffer is sufficient or maxium size is
+ * reached
+ */
+static void kvm_flic_save(QEMUFile *f, void *opaque)
+{
+    KVMS390FLICState *flic = opaque;
+    int len = FLIC_SAVE_INITIAL_SIZE;
+    void *buf;
+    int count;
+
+    flic_disable_wait_pfault((struct KVMS390FLICState *) opaque);
+
+    buf = g_try_malloc0(len);
+    if (!buf) {
+        /* Storing FLIC_FAILED into the count field here will cause the
+         * target system to fail when attempting to load irqs from the
+         * migration state */
+        error_report("flic: couldn't allocate memory");
+        qemu_put_be64(f, FLIC_FAILED);
+        return;
+    }
+
+    count = __get_all_irqs(flic, &buf, len);
+    if (count < 0) {
+        error_report("flic: couldn't retrieve irqs from kernel, rc %d",
+                     count);
+        /* Storing FLIC_FAILED into the count field here will cause the
+         * target system to fail when attempting to load irqs from the
+         * migration state */
+        qemu_put_be64(f, FLIC_FAILED);
+    } else {
+        qemu_put_be64(f, count);
+        qemu_put_buffer(f, (uint8_t *) buf,
+                        count * sizeof(struct kvm_s390_irq));
+    }
+    g_free(buf);
+}
+
+/**
+ * kvm_flic_load - Load pending floating interrupts
+ * @f: QEMUFile containing migration state
+ * @opaque: pointer to flic device state
+ * @version_id: version id for migration
+ *
+ * Returns: value of flic_enqueue_irqs, -EINVAL on error
+ * Note: Do nothing when no interrupts where stored
+ * in QEMUFile
+ */
+static int kvm_flic_load(QEMUFile *f, void *opaque, int version_id)
+{
+    uint64_t len = 0;
+    uint64_t count = 0;
+    void *buf = NULL;
+    int r = 0;
+
+    if (version_id != FLIC_SAVEVM_VERSION) {
+        r = -EINVAL;
+        goto out;
+    }
+
+    flic_enable_pfault((struct KVMS390FLICState *) opaque);
+
+    count = qemu_get_be64(f);
+    len = count * sizeof(struct kvm_s390_irq);
+    if (count == FLIC_FAILED) {
+        r = -EINVAL;
+        goto out;
+    }
+    if (count == 0) {
+        r = 0;
+        goto out;
+    }
+    buf = g_try_malloc0(len);
+    if (!buf) {
+        r = -ENOMEM;
+        goto out;
+    }
+
+    if (qemu_get_buffer(f, (uint8_t *) buf, len) != len) {
+        r = -EINVAL;
+        goto out_free;
+    }
+    r = flic_enqueue_irqs(buf, len, (struct KVMS390FLICState *) opaque);
+
+out_free:
+    g_free(buf);
+out:
+    return r;
+}
+
+static void kvm_s390_flic_realize(DeviceState *dev, Error **errp)
+{
+    KVMS390FLICState *flic_state = KVM_S390_FLIC(dev);
+    struct kvm_create_device cd = {0};
+    int ret;
+
+    flic_state->fd = -1;
+    if (!kvm_check_extension(kvm_state, KVM_CAP_DEVICE_CTRL)) {
+        trace_flic_no_device_api(errno);
+        return;
+    }
+
+    cd.type = KVM_DEV_TYPE_FLIC;
+    ret = kvm_vm_ioctl(kvm_state, KVM_CREATE_DEVICE, &cd);
+    if (ret < 0) {
+        trace_flic_create_device(errno);
+        return;
+    }
+    flic_state->fd = cd.fd;
+
+    /* Register savevm handler for floating interrupts */
+    register_savevm(NULL, "s390-flic", 0, 1, kvm_flic_save,
+                    kvm_flic_load, (void *) flic_state);
+}
+
+static void kvm_s390_flic_unrealize(DeviceState *dev, Error **errp)
+{
+    KVMS390FLICState *flic_state = KVM_S390_FLIC(dev);
+
+    unregister_savevm(DEVICE(flic_state), "s390-flic", flic_state);
+}
+
+static void kvm_s390_flic_reset(DeviceState *dev)
+{
+    KVMS390FLICState *flic = KVM_S390_FLIC(dev);
+    struct kvm_device_attr attr = {
+        .group = KVM_DEV_FLIC_CLEAR_IRQS,
+    };
+    int rc = 0;
+
+    if (flic->fd == -1) {
+        return;
+    }
+
+    flic_disable_wait_pfault(flic);
+
+    rc = ioctl(flic->fd, KVM_SET_DEVICE_ATTR, &attr);
+    if (rc) {
+        trace_flic_reset_failed(errno);
+    }
+
+    flic_enable_pfault(flic);
+}
+
+static void kvm_s390_flic_class_init(ObjectClass *oc, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(oc);
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_CLASS(oc);
+
+    dc->realize = kvm_s390_flic_realize;
+    dc->unrealize = kvm_s390_flic_unrealize;
+    dc->reset = kvm_s390_flic_reset;
+    fsc->register_io_adapter = kvm_s390_register_io_adapter;
+    fsc->io_adapter_map = kvm_s390_io_adapter_map;
+    fsc->add_adapter_routes = kvm_s390_add_adapter_routes;
+    fsc->release_adapter_routes = kvm_s390_release_adapter_routes;
+}
+
+static const TypeInfo kvm_s390_flic_info = {
+    .name          = TYPE_KVM_S390_FLIC,
+    .parent        = TYPE_S390_FLIC_COMMON,
+    .instance_size = sizeof(KVMS390FLICState),
+    .class_init    = kvm_s390_flic_class_init,
+};
+
+static void kvm_s390_flic_register_types(void)
+{
+    type_register_static(&kvm_s390_flic_info);
+}
+
+type_init(kvm_s390_flic_register_types)
diff --git a/hw/lm32/lm32_boards.c b/hw/lm32/lm32_boards.c
index 5e22e9b..0e01340 100644
--- a/hw/lm32/lm32_boards.c
+++ b/hw/lm32/lm32_boards.c
@@ -69,10 +69,10 @@
     env->deba = reset_info->flash_base;
 }
 
-static void lm32_evr_init(QEMUMachineInitArgs *args)
+static void lm32_evr_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
@@ -162,12 +162,12 @@
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static void lm32_uclinux_init(QEMUMachineInitArgs *args)
+static void lm32_uclinux_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
diff --git a/hw/lm32/milkymist.c b/hw/lm32/milkymist.c
index baf234c..81c3933 100644
--- a/hw/lm32/milkymist.c
+++ b/hw/lm32/milkymist.c
@@ -74,12 +74,12 @@
 }
 
 static void
-milkymist_init(QEMUMachineInitArgs *args)
+milkymist_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     int kernel_size;
diff --git a/hw/m68k/an5206.c b/hw/m68k/an5206.c
index 24f2068..684496a 100644
--- a/hw/m68k/an5206.c
+++ b/hw/m68k/an5206.c
@@ -20,11 +20,11 @@
 
 /* Board init.  */
 
-static void an5206_init(QEMUMachineInitArgs *args)
+static void an5206_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     M68kCPU *cpu;
     CPUM68KState *env;
     int kernel_size;
diff --git a/hw/m68k/dummy_m68k.c b/hw/m68k/dummy_m68k.c
index 86e2e6e..6db1b71 100644
--- a/hw/m68k/dummy_m68k.c
+++ b/hw/m68k/dummy_m68k.c
@@ -16,11 +16,11 @@
 
 /* Board init.  */
 
-static void dummy_m68k_init(QEMUMachineInitArgs *args)
+static void dummy_m68k_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     CPUM68KState *env;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/m68k/mcf5208.c b/hw/m68k/mcf5208.c
index 6e30c0b..2ef617f 100644
--- a/hw/m68k/mcf5208.c
+++ b/hw/m68k/mcf5208.c
@@ -188,11 +188,11 @@
     }
 }
 
-static void mcf5208evb_init(QEMUMachineInitArgs *args)
+static void mcf5208evb_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     M68kCPU *cpu;
     CPUM68KState *env;
     int kernel_size;
diff --git a/hw/microblaze/petalogix_ml605_mmu.c b/hw/microblaze/petalogix_ml605_mmu.c
index 40a9f5c..aea9c5b 100644
--- a/hw/microblaze/petalogix_ml605_mmu.c
+++ b/hw/microblaze/petalogix_ml605_mmu.c
@@ -79,9 +79,9 @@
 }
 
 static void
-petalogix_ml605_init(QEMUMachineInitArgs *args)
+petalogix_ml605_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
+    ram_addr_t ram_size = machine->ram_size;
     MemoryRegion *address_space_mem = get_system_memory();
     DeviceState *dev, *dma, *eth0;
     Object *ds, *cs;
@@ -196,13 +196,13 @@
             qemu_irq cs_line;
 
             dev = ssi_create_slave(spi, "n25q128");
-            cs_line = qdev_get_gpio_in(dev, 0);
+            cs_line = qdev_get_gpio_in_named(dev, SSI_GPIO_CS, 0);
             sysbus_connect_irq(busdev, i+1, cs_line);
         }
     }
 
     microblaze_load_kernel(cpu, ddr_base, ram_size,
-                           args->initrd_filename,
+                           machine->initrd_filename,
                            BINARY_DEVICE_TREE_FILE,
                            machine_cpu_reset);
 
diff --git a/hw/microblaze/petalogix_s3adsp1800_mmu.c b/hw/microblaze/petalogix_s3adsp1800_mmu.c
index 6c45e20..49dc6d1 100644
--- a/hw/microblaze/petalogix_s3adsp1800_mmu.c
+++ b/hw/microblaze/petalogix_s3adsp1800_mmu.c
@@ -59,10 +59,10 @@
 }
 
 static void
-petalogix_s3adsp1800_init(QEMUMachineInitArgs *args)
+petalogix_s3adsp1800_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
     DeviceState *dev;
     MicroBlazeCPU *cpu;
     DriveInfo *dinfo;
@@ -128,7 +128,7 @@
     sysbus_connect_irq(SYS_BUS_DEVICE(dev), 0, irq[ETHLITE_IRQ]);
 
     microblaze_load_kernel(cpu, ddr_base, ram_size,
-                           args->initrd_filename,
+                           machine->initrd_filename,
                            BINARY_DEVICE_TREE_FILE,
                            machine_cpu_reset);
 }
diff --git a/hw/mips/mips_fulong2e.c b/hw/mips/mips_fulong2e.c
index 30d9f19..f7533ed 100644
--- a/hw/mips/mips_fulong2e.c
+++ b/hw/mips/mips_fulong2e.c
@@ -178,24 +178,24 @@
     /* Small bootloader */
     p = (uint32_t *) base;
 
-    stl_raw(p++, 0x0bf00010);                                      /* j 0x1fc00040 */
-    stl_raw(p++, 0x00000000);                                      /* nop */
+    stl_p(p++, 0x0bf00010);                                      /* j 0x1fc00040 */
+    stl_p(p++, 0x00000000);                                      /* nop */
 
     /* Second part of the bootloader */
     p = (uint32_t *) (base + 0x040);
 
-    stl_raw(p++, 0x3c040000);                                      /* lui a0, 0 */
-    stl_raw(p++, 0x34840002);                                      /* ori a0, a0, 2 */
-    stl_raw(p++, 0x3c050000 | ((ENVP_ADDR >> 16) & 0xffff));       /* lui a1, high(ENVP_ADDR) */
-    stl_raw(p++, 0x34a50000 | (ENVP_ADDR & 0xffff));               /* ori a1, a0, low(ENVP_ADDR) */
-    stl_raw(p++, 0x3c060000 | (((ENVP_ADDR + 8) >> 16) & 0xffff)); /* lui a2, high(ENVP_ADDR + 8) */
-    stl_raw(p++, 0x34c60000 | ((ENVP_ADDR + 8) & 0xffff));         /* ori a2, a2, low(ENVP_ADDR + 8) */
-    stl_raw(p++, 0x3c070000 | (loaderparams.ram_size >> 16));      /* lui a3, high(env->ram_size) */
-    stl_raw(p++, 0x34e70000 | (loaderparams.ram_size & 0xffff));   /* ori a3, a3, low(env->ram_size) */
-    stl_raw(p++, 0x3c1f0000 | ((kernel_addr >> 16) & 0xffff));     /* lui ra, high(kernel_addr) */;
-    stl_raw(p++, 0x37ff0000 | (kernel_addr & 0xffff));             /* ori ra, ra, low(kernel_addr) */
-    stl_raw(p++, 0x03e00008);                                      /* jr ra */
-    stl_raw(p++, 0x00000000);                                      /* nop */
+    stl_p(p++, 0x3c040000);                                      /* lui a0, 0 */
+    stl_p(p++, 0x34840002);                                      /* ori a0, a0, 2 */
+    stl_p(p++, 0x3c050000 | ((ENVP_ADDR >> 16) & 0xffff));       /* lui a1, high(ENVP_ADDR) */
+    stl_p(p++, 0x34a50000 | (ENVP_ADDR & 0xffff));               /* ori a1, a0, low(ENVP_ADDR) */
+    stl_p(p++, 0x3c060000 | (((ENVP_ADDR + 8) >> 16) & 0xffff)); /* lui a2, high(ENVP_ADDR + 8) */
+    stl_p(p++, 0x34c60000 | ((ENVP_ADDR + 8) & 0xffff));         /* ori a2, a2, low(ENVP_ADDR + 8) */
+    stl_p(p++, 0x3c070000 | (loaderparams.ram_size >> 16));      /* lui a3, high(env->ram_size) */
+    stl_p(p++, 0x34e70000 | (loaderparams.ram_size & 0xffff));   /* ori a3, a3, low(env->ram_size) */
+    stl_p(p++, 0x3c1f0000 | ((kernel_addr >> 16) & 0xffff));     /* lui ra, high(kernel_addr) */;
+    stl_p(p++, 0x37ff0000 | (kernel_addr & 0xffff));             /* ori ra, ra, low(kernel_addr) */
+    stl_p(p++, 0x03e00008);                                      /* jr ra */
+    stl_p(p++, 0x00000000);                                      /* nop */
 }
 
 
@@ -259,13 +259,13 @@
     }
 }
 
-static void mips_fulong2e_init(QEMUMachineInitArgs *args)
+static void mips_fulong2e_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/mips/mips_jazz.c b/hw/mips/mips_jazz.c
index 5f6dd9f..c113a80 100644
--- a/hw/mips/mips_jazz.c
+++ b/hw/mips/mips_jazz.c
@@ -329,19 +329,19 @@
 }
 
 static
-void mips_magnum_init(QEMUMachineInitArgs *args)
+void mips_magnum_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
         mips_jazz_init(get_system_memory(), get_system_io(),
                        ram_size, cpu_model, JAZZ_MAGNUM);
 }
 
 static
-void mips_pica61_init(QEMUMachineInitArgs *args)
+void mips_pica61_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
     mips_jazz_init(get_system_memory(), get_system_io(),
                    ram_size, cpu_model, JAZZ_PICA61);
 }
diff --git a/hw/mips/mips_malta.c b/hw/mips/mips_malta.c
index ac5ec44..f4a7d47 100644
--- a/hw/mips/mips_malta.c
+++ b/hw/mips/mips_malta.c
@@ -609,136 +609,136 @@
 
     /* Small bootloader */
     p = (uint32_t *)base;
-    stl_raw(p++, 0x0bf00160);                                      /* j 0x1fc00580 */
-    stl_raw(p++, 0x00000000);                                      /* nop */
+    stl_p(p++, 0x0bf00160);                                      /* j 0x1fc00580 */
+    stl_p(p++, 0x00000000);                                      /* nop */
 
     /* YAMON service vector */
-    stl_raw(base + 0x500, 0xbfc00580);      /* start: */
-    stl_raw(base + 0x504, 0xbfc0083c);      /* print_count: */
-    stl_raw(base + 0x520, 0xbfc00580);      /* start: */
-    stl_raw(base + 0x52c, 0xbfc00800);      /* flush_cache: */
-    stl_raw(base + 0x534, 0xbfc00808);      /* print: */
-    stl_raw(base + 0x538, 0xbfc00800);      /* reg_cpu_isr: */
-    stl_raw(base + 0x53c, 0xbfc00800);      /* unred_cpu_isr: */
-    stl_raw(base + 0x540, 0xbfc00800);      /* reg_ic_isr: */
-    stl_raw(base + 0x544, 0xbfc00800);      /* unred_ic_isr: */
-    stl_raw(base + 0x548, 0xbfc00800);      /* reg_esr: */
-    stl_raw(base + 0x54c, 0xbfc00800);      /* unreg_esr: */
-    stl_raw(base + 0x550, 0xbfc00800);      /* getchar: */
-    stl_raw(base + 0x554, 0xbfc00800);      /* syscon_read: */
+    stl_p(base + 0x500, 0xbfc00580);      /* start: */
+    stl_p(base + 0x504, 0xbfc0083c);      /* print_count: */
+    stl_p(base + 0x520, 0xbfc00580);      /* start: */
+    stl_p(base + 0x52c, 0xbfc00800);      /* flush_cache: */
+    stl_p(base + 0x534, 0xbfc00808);      /* print: */
+    stl_p(base + 0x538, 0xbfc00800);      /* reg_cpu_isr: */
+    stl_p(base + 0x53c, 0xbfc00800);      /* unred_cpu_isr: */
+    stl_p(base + 0x540, 0xbfc00800);      /* reg_ic_isr: */
+    stl_p(base + 0x544, 0xbfc00800);      /* unred_ic_isr: */
+    stl_p(base + 0x548, 0xbfc00800);      /* reg_esr: */
+    stl_p(base + 0x54c, 0xbfc00800);      /* unreg_esr: */
+    stl_p(base + 0x550, 0xbfc00800);      /* getchar: */
+    stl_p(base + 0x554, 0xbfc00800);      /* syscon_read: */
 
 
     /* Second part of the bootloader */
     p = (uint32_t *) (base + 0x580);
-    stl_raw(p++, 0x24040002);                                      /* addiu a0, zero, 2 */
-    stl_raw(p++, 0x3c1d0000 | (((ENVP_ADDR - 64) >> 16) & 0xffff)); /* lui sp, high(ENVP_ADDR) */
-    stl_raw(p++, 0x37bd0000 | ((ENVP_ADDR - 64) & 0xffff));        /* ori sp, sp, low(ENVP_ADDR) */
-    stl_raw(p++, 0x3c050000 | ((ENVP_ADDR >> 16) & 0xffff));       /* lui a1, high(ENVP_ADDR) */
-    stl_raw(p++, 0x34a50000 | (ENVP_ADDR & 0xffff));               /* ori a1, a1, low(ENVP_ADDR) */
-    stl_raw(p++, 0x3c060000 | (((ENVP_ADDR + 8) >> 16) & 0xffff)); /* lui a2, high(ENVP_ADDR + 8) */
-    stl_raw(p++, 0x34c60000 | ((ENVP_ADDR + 8) & 0xffff));         /* ori a2, a2, low(ENVP_ADDR + 8) */
-    stl_raw(p++, 0x3c070000 | (loaderparams.ram_size >> 16));     /* lui a3, high(ram_size) */
-    stl_raw(p++, 0x34e70000 | (loaderparams.ram_size & 0xffff));  /* ori a3, a3, low(ram_size) */
+    stl_p(p++, 0x24040002);                                      /* addiu a0, zero, 2 */
+    stl_p(p++, 0x3c1d0000 | (((ENVP_ADDR - 64) >> 16) & 0xffff)); /* lui sp, high(ENVP_ADDR) */
+    stl_p(p++, 0x37bd0000 | ((ENVP_ADDR - 64) & 0xffff));        /* ori sp, sp, low(ENVP_ADDR) */
+    stl_p(p++, 0x3c050000 | ((ENVP_ADDR >> 16) & 0xffff));       /* lui a1, high(ENVP_ADDR) */
+    stl_p(p++, 0x34a50000 | (ENVP_ADDR & 0xffff));               /* ori a1, a1, low(ENVP_ADDR) */
+    stl_p(p++, 0x3c060000 | (((ENVP_ADDR + 8) >> 16) & 0xffff)); /* lui a2, high(ENVP_ADDR + 8) */
+    stl_p(p++, 0x34c60000 | ((ENVP_ADDR + 8) & 0xffff));         /* ori a2, a2, low(ENVP_ADDR + 8) */
+    stl_p(p++, 0x3c070000 | (loaderparams.ram_size >> 16));     /* lui a3, high(ram_size) */
+    stl_p(p++, 0x34e70000 | (loaderparams.ram_size & 0xffff));  /* ori a3, a3, low(ram_size) */
 
     /* Load BAR registers as done by YAMON */
-    stl_raw(p++, 0x3c09b400);                                      /* lui t1, 0xb400 */
+    stl_p(p++, 0x3c09b400);                                      /* lui t1, 0xb400 */
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c08df00);                                      /* lui t0, 0xdf00 */
+    stl_p(p++, 0x3c08df00);                                      /* lui t0, 0xdf00 */
 #else
-    stl_raw(p++, 0x340800df);                                      /* ori t0, r0, 0x00df */
+    stl_p(p++, 0x340800df);                                      /* ori t0, r0, 0x00df */
 #endif
-    stl_raw(p++, 0xad280068);                                      /* sw t0, 0x0068(t1) */
+    stl_p(p++, 0xad280068);                                      /* sw t0, 0x0068(t1) */
 
-    stl_raw(p++, 0x3c09bbe0);                                      /* lui t1, 0xbbe0 */
+    stl_p(p++, 0x3c09bbe0);                                      /* lui t1, 0xbbe0 */
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c08c000);                                      /* lui t0, 0xc000 */
+    stl_p(p++, 0x3c08c000);                                      /* lui t0, 0xc000 */
 #else
-    stl_raw(p++, 0x340800c0);                                      /* ori t0, r0, 0x00c0 */
+    stl_p(p++, 0x340800c0);                                      /* ori t0, r0, 0x00c0 */
 #endif
-    stl_raw(p++, 0xad280048);                                      /* sw t0, 0x0048(t1) */
+    stl_p(p++, 0xad280048);                                      /* sw t0, 0x0048(t1) */
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c084000);                                      /* lui t0, 0x4000 */
+    stl_p(p++, 0x3c084000);                                      /* lui t0, 0x4000 */
 #else
-    stl_raw(p++, 0x34080040);                                      /* ori t0, r0, 0x0040 */
+    stl_p(p++, 0x34080040);                                      /* ori t0, r0, 0x0040 */
 #endif
-    stl_raw(p++, 0xad280050);                                      /* sw t0, 0x0050(t1) */
+    stl_p(p++, 0xad280050);                                      /* sw t0, 0x0050(t1) */
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c088000);                                      /* lui t0, 0x8000 */
+    stl_p(p++, 0x3c088000);                                      /* lui t0, 0x8000 */
 #else
-    stl_raw(p++, 0x34080080);                                      /* ori t0, r0, 0x0080 */
+    stl_p(p++, 0x34080080);                                      /* ori t0, r0, 0x0080 */
 #endif
-    stl_raw(p++, 0xad280058);                                      /* sw t0, 0x0058(t1) */
+    stl_p(p++, 0xad280058);                                      /* sw t0, 0x0058(t1) */
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c083f00);                                      /* lui t0, 0x3f00 */
+    stl_p(p++, 0x3c083f00);                                      /* lui t0, 0x3f00 */
 #else
-    stl_raw(p++, 0x3408003f);                                      /* ori t0, r0, 0x003f */
+    stl_p(p++, 0x3408003f);                                      /* ori t0, r0, 0x003f */
 #endif
-    stl_raw(p++, 0xad280060);                                      /* sw t0, 0x0060(t1) */
+    stl_p(p++, 0xad280060);                                      /* sw t0, 0x0060(t1) */
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c08c100);                                      /* lui t0, 0xc100 */
+    stl_p(p++, 0x3c08c100);                                      /* lui t0, 0xc100 */
 #else
-    stl_raw(p++, 0x340800c1);                                      /* ori t0, r0, 0x00c1 */
+    stl_p(p++, 0x340800c1);                                      /* ori t0, r0, 0x00c1 */
 #endif
-    stl_raw(p++, 0xad280080);                                      /* sw t0, 0x0080(t1) */
+    stl_p(p++, 0xad280080);                                      /* sw t0, 0x0080(t1) */
 #ifdef TARGET_WORDS_BIGENDIAN
-    stl_raw(p++, 0x3c085e00);                                      /* lui t0, 0x5e00 */
+    stl_p(p++, 0x3c085e00);                                      /* lui t0, 0x5e00 */
 #else
-    stl_raw(p++, 0x3408005e);                                      /* ori t0, r0, 0x005e */
+    stl_p(p++, 0x3408005e);                                      /* ori t0, r0, 0x005e */
 #endif
-    stl_raw(p++, 0xad280088);                                      /* sw t0, 0x0088(t1) */
+    stl_p(p++, 0xad280088);                                      /* sw t0, 0x0088(t1) */
 
     /* Jump to kernel code */
-    stl_raw(p++, 0x3c1f0000 | ((kernel_entry >> 16) & 0xffff));    /* lui ra, high(kernel_entry) */
-    stl_raw(p++, 0x37ff0000 | (kernel_entry & 0xffff));            /* ori ra, ra, low(kernel_entry) */
-    stl_raw(p++, 0x03e00008);                                      /* jr ra */
-    stl_raw(p++, 0x00000000);                                      /* nop */
+    stl_p(p++, 0x3c1f0000 | ((kernel_entry >> 16) & 0xffff));    /* lui ra, high(kernel_entry) */
+    stl_p(p++, 0x37ff0000 | (kernel_entry & 0xffff));            /* ori ra, ra, low(kernel_entry) */
+    stl_p(p++, 0x03e00008);                                      /* jr ra */
+    stl_p(p++, 0x00000000);                                      /* nop */
 
     /* YAMON subroutines */
     p = (uint32_t *) (base + 0x800);
-    stl_raw(p++, 0x03e00008);                                     /* jr ra */
-    stl_raw(p++, 0x24020000);                                     /* li v0,0 */
+    stl_p(p++, 0x03e00008);                                     /* jr ra */
+    stl_p(p++, 0x24020000);                                     /* li v0,0 */
    /* 808 YAMON print */
-    stl_raw(p++, 0x03e06821);                                     /* move t5,ra */
-    stl_raw(p++, 0x00805821);                                     /* move t3,a0 */
-    stl_raw(p++, 0x00a05021);                                     /* move t2,a1 */
-    stl_raw(p++, 0x91440000);                                     /* lbu a0,0(t2) */
-    stl_raw(p++, 0x254a0001);                                     /* addiu t2,t2,1 */
-    stl_raw(p++, 0x10800005);                                     /* beqz a0,834 */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x0ff0021c);                                     /* jal 870 */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x08000205);                                     /* j 814 */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x01a00008);                                     /* jr t5 */
-    stl_raw(p++, 0x01602021);                                     /* move a0,t3 */
+    stl_p(p++, 0x03e06821);                                     /* move t5,ra */
+    stl_p(p++, 0x00805821);                                     /* move t3,a0 */
+    stl_p(p++, 0x00a05021);                                     /* move t2,a1 */
+    stl_p(p++, 0x91440000);                                     /* lbu a0,0(t2) */
+    stl_p(p++, 0x254a0001);                                     /* addiu t2,t2,1 */
+    stl_p(p++, 0x10800005);                                     /* beqz a0,834 */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x0ff0021c);                                     /* jal 870 */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x08000205);                                     /* j 814 */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x01a00008);                                     /* jr t5 */
+    stl_p(p++, 0x01602021);                                     /* move a0,t3 */
     /* 0x83c YAMON print_count */
-    stl_raw(p++, 0x03e06821);                                     /* move t5,ra */
-    stl_raw(p++, 0x00805821);                                     /* move t3,a0 */
-    stl_raw(p++, 0x00a05021);                                     /* move t2,a1 */
-    stl_raw(p++, 0x00c06021);                                     /* move t4,a2 */
-    stl_raw(p++, 0x91440000);                                     /* lbu a0,0(t2) */
-    stl_raw(p++, 0x0ff0021c);                                     /* jal 870 */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x254a0001);                                     /* addiu t2,t2,1 */
-    stl_raw(p++, 0x258cffff);                                     /* addiu t4,t4,-1 */
-    stl_raw(p++, 0x1580fffa);                                     /* bnez t4,84c */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x01a00008);                                     /* jr t5 */
-    stl_raw(p++, 0x01602021);                                     /* move a0,t3 */
+    stl_p(p++, 0x03e06821);                                     /* move t5,ra */
+    stl_p(p++, 0x00805821);                                     /* move t3,a0 */
+    stl_p(p++, 0x00a05021);                                     /* move t2,a1 */
+    stl_p(p++, 0x00c06021);                                     /* move t4,a2 */
+    stl_p(p++, 0x91440000);                                     /* lbu a0,0(t2) */
+    stl_p(p++, 0x0ff0021c);                                     /* jal 870 */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x254a0001);                                     /* addiu t2,t2,1 */
+    stl_p(p++, 0x258cffff);                                     /* addiu t4,t4,-1 */
+    stl_p(p++, 0x1580fffa);                                     /* bnez t4,84c */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x01a00008);                                     /* jr t5 */
+    stl_p(p++, 0x01602021);                                     /* move a0,t3 */
     /* 0x870 */
-    stl_raw(p++, 0x3c08b800);                                     /* lui t0,0xb400 */
-    stl_raw(p++, 0x350803f8);                                     /* ori t0,t0,0x3f8 */
-    stl_raw(p++, 0x91090005);                                     /* lbu t1,5(t0) */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x31290040);                                     /* andi t1,t1,0x40 */
-    stl_raw(p++, 0x1120fffc);                                     /* beqz t1,878 <outch+0x8> */
-    stl_raw(p++, 0x00000000);                                     /* nop */
-    stl_raw(p++, 0x03e00008);                                     /* jr ra */
-    stl_raw(p++, 0xa1040000);                                     /* sb a0,0(t0) */
+    stl_p(p++, 0x3c08b800);                                     /* lui t0,0xb400 */
+    stl_p(p++, 0x350803f8);                                     /* ori t0,t0,0x3f8 */
+    stl_p(p++, 0x91090005);                                     /* lbu t1,5(t0) */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x31290040);                                     /* andi t1,t1,0x40 */
+    stl_p(p++, 0x1120fffc);                                     /* beqz t1,878 <outch+0x8> */
+    stl_p(p++, 0x00000000);                                     /* nop */
+    stl_p(p++, 0x03e00008);                                     /* jr ra */
+    stl_p(p++, 0xa1040000);                                     /* sb a0,0(t0) */
 
 }
 
@@ -875,13 +875,13 @@
 }
 
 static
-void mips_malta_init(QEMUMachineInitArgs *args)
+void mips_malta_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     pflash_t *fl;
     MemoryRegion *system_memory = get_system_memory();
diff --git a/hw/mips/mips_mipssim.c b/hw/mips/mips_mipssim.c
index 239aa6a..413e64d 100644
--- a/hw/mips/mips_mipssim.c
+++ b/hw/mips/mips_mipssim.c
@@ -133,13 +133,13 @@
 }
 
 static void
-mips_mipssim_init(QEMUMachineInitArgs *args)
+mips_mipssim_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *isa = g_new(MemoryRegion, 1);
diff --git a/hw/mips/mips_r4k.c b/hw/mips/mips_r4k.c
index e94b543..7120293 100644
--- a/hw/mips/mips_r4k.c
+++ b/hw/mips/mips_r4k.c
@@ -153,13 +153,13 @@
 
 static const int sector_len = 32 * 1024;
 static
-void mips_r4k_init(QEMUMachineInitArgs *args)
+void mips_r4k_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/misc/Makefile.objs b/hw/misc/Makefile.objs
index f674365..979e532 100644
--- a/hw/misc/Makefile.objs
+++ b/hw/misc/Makefile.objs
@@ -29,7 +29,6 @@
 obj-$(CONFIG_ECCMEMCTL) += eccmemctl.o
 obj-$(CONFIG_EXYNOS4) += exynos4210_pmu.o
 obj-$(CONFIG_IMX) += imx_ccm.o
-obj-$(CONFIG_LM32) += lm32_sys.o
 obj-$(CONFIG_MILKYMIST) += milkymist-hpdmc.o
 obj-$(CONFIG_MILKYMIST) += milkymist-pfpu.o
 obj-$(CONFIG_MAINSTONE) += mst_fpga.o
diff --git a/hw/misc/lm32_sys.c b/hw/misc/lm32_sys.c
deleted file mode 100644
index 778eb6e..0000000
--- a/hw/misc/lm32_sys.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- *  QEMU model of the LatticeMico32 system control block.
- *
- *  Copyright (c) 2010 Michael Walle <michael@walle.cc>
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-/*
- * This model is mainly intended for testing purposes and doesn't fit to any
- * real hardware. On the one hand it provides a control register (R_CTRL) on
- * the other hand it supports the lm32 tests.
- *
- * A write to the control register causes a system shutdown.
- * Tests first write the pointer to a test name to the test name register
- * (R_TESTNAME) and then write a zero to the pass/fail register (R_PASSFAIL) if
- * the test is passed or any non-zero value to it if the test is failed.
- */
-
-#include "hw/hw.h"
-#include "hw/sysbus.h"
-#include "trace.h"
-#include "qemu/log.h"
-#include "qemu/error-report.h"
-#include "sysemu/sysemu.h"
-
-enum {
-    R_CTRL = 0,
-    R_PASSFAIL,
-    R_TESTNAME,
-    R_MAX
-};
-
-#define MAX_TESTNAME_LEN 32
-
-#define TYPE_LM32_SYS "lm32-sys"
-#define LM32_SYS(obj) OBJECT_CHECK(LM32SysState, (obj), TYPE_LM32_SYS)
-
-struct LM32SysState {
-    SysBusDevice parent_obj;
-
-    MemoryRegion iomem;
-    uint32_t base;
-    uint32_t regs[R_MAX];
-    uint8_t testname[MAX_TESTNAME_LEN];
-};
-typedef struct LM32SysState LM32SysState;
-
-static void copy_testname(LM32SysState *s)
-{
-    cpu_physical_memory_read(s->regs[R_TESTNAME], s->testname,
-            MAX_TESTNAME_LEN);
-    s->testname[MAX_TESTNAME_LEN - 1] = '\0';
-}
-
-static void sys_write(void *opaque, hwaddr addr,
-                      uint64_t value, unsigned size)
-{
-    LM32SysState *s = opaque;
-    char *testname;
-
-    trace_lm32_sys_memory_write(addr, value);
-
-    addr >>= 2;
-    switch (addr) {
-    case R_CTRL:
-        qemu_system_shutdown_request();
-        break;
-    case R_PASSFAIL:
-        s->regs[addr] = value;
-        testname = (char *)s->testname;
-        fprintf(stderr, "TC  %-*s %s\n", MAX_TESTNAME_LEN,
-                testname, (value) ? "FAILED" : "OK");
-        if (value) {
-            cpu_dump_state(qemu_get_cpu(0), stderr, fprintf, 0);
-        }
-        break;
-    case R_TESTNAME:
-        s->regs[addr] = value;
-        copy_testname(s);
-        break;
-
-    default:
-        error_report("lm32_sys: write access to unknown register 0x"
-                TARGET_FMT_plx, addr << 2);
-        break;
-    }
-}
-
-static bool sys_ops_accepts(void *opaque, hwaddr addr,
-                            unsigned size, bool is_write)
-{
-    return is_write && size == 4;
-}
-
-static const MemoryRegionOps sys_ops = {
-    .write = sys_write,
-    .valid.accepts = sys_ops_accepts,
-    .endianness = DEVICE_NATIVE_ENDIAN,
-};
-
-static void sys_reset(DeviceState *d)
-{
-    LM32SysState *s = LM32_SYS(d);
-    int i;
-
-    for (i = 0; i < R_MAX; i++) {
-        s->regs[i] = 0;
-    }
-    memset(s->testname, 0, MAX_TESTNAME_LEN);
-}
-
-static int lm32_sys_init(SysBusDevice *dev)
-{
-    LM32SysState *s = LM32_SYS(dev);
-
-    memory_region_init_io(&s->iomem, OBJECT(dev), &sys_ops , s,
-                          "sys", R_MAX * 4);
-    sysbus_init_mmio(dev, &s->iomem);
-
-    /* Note: This device is not created in the board initialization,
-     * instead it has to be added with the -device parameter. Therefore,
-     * the device maps itself. */
-    sysbus_mmio_map(dev, 0, s->base);
-
-    return 0;
-}
-
-static const VMStateDescription vmstate_lm32_sys = {
-    .name = "lm32-sys",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .fields = (VMStateField[]) {
-        VMSTATE_UINT32_ARRAY(regs, LM32SysState, R_MAX),
-        VMSTATE_BUFFER(testname, LM32SysState),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
-static Property lm32_sys_properties[] = {
-    DEFINE_PROP_UINT32("base", LM32SysState, base, 0xffff0000),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void lm32_sys_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
-
-    k->init = lm32_sys_init;
-    dc->reset = sys_reset;
-    dc->vmsd = &vmstate_lm32_sys;
-    dc->props = lm32_sys_properties;
-}
-
-static const TypeInfo lm32_sys_info = {
-    .name          = TYPE_LM32_SYS,
-    .parent        = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(LM32SysState),
-    .class_init    = lm32_sys_class_init,
-};
-
-static void lm32_sys_register_types(void)
-{
-    type_register_static(&lm32_sys_info);
-}
-
-type_init(lm32_sys_register_types)
diff --git a/hw/misc/vfio.c b/hw/misc/vfio.c
index 9cf5b84..7437c2e 100644
--- a/hw/misc/vfio.c
+++ b/hw/misc/vfio.c
@@ -133,6 +133,15 @@
     VFIO_INT_MSIX = 3,
 };
 
+typedef struct VFIOAddressSpace {
+    AddressSpace *as;
+    QLIST_HEAD(, VFIOContainer) containers;
+    QLIST_ENTRY(VFIOAddressSpace) list;
+} VFIOAddressSpace;
+
+static QLIST_HEAD(, VFIOAddressSpace) vfio_address_spaces =
+    QLIST_HEAD_INITIALIZER(vfio_address_spaces);
+
 struct VFIOGroup;
 
 typedef struct VFIOType1 {
@@ -142,6 +151,7 @@
 } VFIOType1;
 
 typedef struct VFIOContainer {
+    VFIOAddressSpace *space;
     int fd; /* /dev/vfio/vfio, empowered by the attached groups */
     struct {
         /* enable abstraction to support various iommu backends */
@@ -150,10 +160,18 @@
         };
         void (*release)(struct VFIOContainer *);
     } iommu_data;
+    QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
     QLIST_HEAD(, VFIOGroup) group_list;
     QLIST_ENTRY(VFIOContainer) next;
 } VFIOContainer;
 
+typedef struct VFIOGuestIOMMU {
+    VFIOContainer *container;
+    MemoryRegion *iommu;
+    Notifier n;
+    QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
+} VFIOGuestIOMMU;
+
 /* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
 typedef struct VFIOMSIXInfo {
     uint8_t table_bar;
@@ -234,9 +252,6 @@
 
 #define MSIX_CAP_LENGTH 12
 
-static QLIST_HEAD(, VFIOContainer)
-    container_list = QLIST_HEAD_INITIALIZER(container_list);
-
 static QLIST_HEAD(, VFIOGroup)
     group_list = QLIST_HEAD_INITIALIZER(group_list);
 
@@ -1668,6 +1683,149 @@
             vdev->host.function);
 }
 
+#define PCI_VENDOR_ID_REALTEK 0x10ec
+
+/*
+ * RTL8168 devices have a backdoor that can access the MSI-X table.  At BAR2
+ * offset 0x70 there is a dword data register, offset 0x74 is a dword address
+ * register.  According to the Linux r8169 driver, the MSI-X table is addressed
+ * when the "type" portion of the address register is set to 0x1.  This appears
+ * to be bits 16:30.  Bit 31 is both a write indicator and some sort of
+ * "address latched" indicator.  Bits 12:15 are a mask field, which we can
+ * ignore because the MSI-X table should always be accessed as a dword (full
+ * mask).  Bits 0:11 is offset within the type.
+ *
+ * Example trace:
+ *
+ * Read from MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data
+ *
+ * Write 0xfee00000 to MSI-X table offset 0
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data
+ * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write
+ * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete
+ */
+
+static uint64_t vfio_rtl8168_window_quirk_read(void *opaque,
+                                               hwaddr addr, unsigned size)
+{
+    VFIOQuirk *quirk = opaque;
+    VFIODevice *vdev = quirk->vdev;
+
+    switch (addr) {
+    case 4: /* address */
+        if (quirk->data.flags) {
+            DPRINTF("%s fake read(%04x:%02x:%02x.%d)\n",
+                    memory_region_name(&quirk->mem), vdev->host.domain,
+                    vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+            return quirk->data.address_match ^ 0x10000000U;
+        }
+        break;
+    case 0: /* data */
+        if (quirk->data.flags) {
+            uint64_t val;
+
+            DPRINTF("%s MSI-X table read(%04x:%02x:%02x.%d)\n",
+                    memory_region_name(&quirk->mem), vdev->host.domain,
+                    vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+            if (!(vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) {
+                return 0;
+            }
+
+            io_mem_read(&vdev->pdev.msix_table_mmio,
+                        (hwaddr)(quirk->data.address_match & 0xfff),
+                        &val, size);
+            return val;
+        }
+    }
+
+    DPRINTF("%s direct read(%04x:%02x:%02x.%d)\n",
+            memory_region_name(&quirk->mem), vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    return vfio_bar_read(&vdev->bars[quirk->data.bar], addr + 0x70, size);
+}
+
+static void vfio_rtl8168_window_quirk_write(void *opaque, hwaddr addr,
+                                            uint64_t data, unsigned size)
+{
+    VFIOQuirk *quirk = opaque;
+    VFIODevice *vdev = quirk->vdev;
+
+    switch (addr) {
+    case 4: /* address */
+        if ((data & 0x7fff0000) == 0x10000) {
+            if (data & 0x10000000U &&
+                vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) {
+
+                DPRINTF("%s MSI-X table write(%04x:%02x:%02x.%d)\n",
+                        memory_region_name(&quirk->mem), vdev->host.domain,
+                        vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+                io_mem_write(&vdev->pdev.msix_table_mmio,
+                             (hwaddr)(quirk->data.address_match & 0xfff),
+                             data, size);
+            }
+
+            quirk->data.flags = 1;
+            quirk->data.address_match = data;
+
+            return;
+        }
+        quirk->data.flags = 0;
+        break;
+    case 0: /* data */
+        quirk->data.address_mask = data;
+        break;
+    }
+
+    DPRINTF("%s direct write(%04x:%02x:%02x.%d)\n",
+            memory_region_name(&quirk->mem), vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    vfio_bar_write(&vdev->bars[quirk->data.bar], addr + 0x70, data, size);
+}
+
+static const MemoryRegionOps vfio_rtl8168_window_quirk = {
+    .read = vfio_rtl8168_window_quirk_read,
+    .write = vfio_rtl8168_window_quirk_write,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+        .unaligned = false,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static void vfio_probe_rtl8168_bar2_window_quirk(VFIODevice *vdev, int nr)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    VFIOQuirk *quirk;
+
+    if (pci_get_word(pdev->config + PCI_VENDOR_ID) != PCI_VENDOR_ID_REALTEK ||
+        pci_get_word(pdev->config + PCI_DEVICE_ID) != 0x8168 || nr != 2) {
+        return;
+    }
+
+    quirk = g_malloc0(sizeof(*quirk));
+    quirk->vdev = vdev;
+    quirk->data.bar = nr;
+
+    memory_region_init_io(&quirk->mem, OBJECT(vdev), &vfio_rtl8168_window_quirk,
+                          quirk, "vfio-rtl8168-window-quirk", 8);
+    memory_region_add_subregion_overlap(&vdev->bars[nr].mem,
+                                        0x70, &quirk->mem, 1);
+
+    QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next);
+
+    DPRINTF("Enabled RTL8168 BAR2 window quirk for device %04x:%02x:%02x.%x\n",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function);
+}
 /*
  * Trap the BAR2 MMIO window to config space as well.
  */
@@ -2071,6 +2229,7 @@
     vfio_probe_nvidia_bar5_window_quirk(vdev, nr);
     vfio_probe_nvidia_bar0_88000_quirk(vdev, nr);
     vfio_probe_nvidia_bar0_1800_quirk(vdev, nr);
+    vfio_probe_rtl8168_bar2_window_quirk(vdev, nr);
 }
 
 static void vfio_bar_quirk_teardown(VFIODevice *vdev, int nr)
@@ -2232,7 +2391,8 @@
 
 static bool vfio_listener_skipped_section(MemoryRegionSection *section)
 {
-    return !memory_region_is_ram(section->mr) ||
+    return (!memory_region_is_ram(section->mr) &&
+            !memory_region_is_iommu(section->mr)) ||
            /*
             * Sizing an enabled 64-bit BAR can cause spurious mappings to
             * addresses in the upper part of the 64-bit address space.  These
@@ -2242,17 +2402,75 @@
            section->offset_within_address_space & (1ULL << 63);
 }
 
+static void vfio_iommu_map_notify(Notifier *n, void *data)
+{
+    VFIOGuestIOMMU *giommu = container_of(n, VFIOGuestIOMMU, n);
+    VFIOContainer *container = giommu->container;
+    IOMMUTLBEntry *iotlb = data;
+    MemoryRegion *mr;
+    hwaddr xlat;
+    hwaddr len = iotlb->addr_mask + 1;
+    void *vaddr;
+    int ret;
+
+    DPRINTF("iommu map @ %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+            iotlb->iova, iotlb->iova + iotlb->addr_mask);
+
+    /*
+     * The IOMMU TLB entry we have just covers translation through
+     * this IOMMU to its immediate target.  We need to translate
+     * it the rest of the way through to memory.
+     */
+    mr = address_space_translate(&address_space_memory,
+                                 iotlb->translated_addr,
+                                 &xlat, &len, iotlb->perm & IOMMU_WO);
+    if (!memory_region_is_ram(mr)) {
+        DPRINTF("iommu map to non memory area %"HWADDR_PRIx"\n",
+                xlat);
+        return;
+    }
+    /*
+     * Translation truncates length to the IOMMU page size,
+     * check that it did not truncate too much.
+     */
+    if (len & iotlb->addr_mask) {
+        DPRINTF("iommu has granularity incompatible with target AS\n");
+        return;
+    }
+
+    if (iotlb->perm != IOMMU_NONE) {
+        vaddr = memory_region_get_ram_ptr(mr) + xlat;
+
+        ret = vfio_dma_map(container, iotlb->iova,
+                           iotlb->addr_mask + 1, vaddr,
+                           !(iotlb->perm & IOMMU_WO) || mr->readonly);
+        if (ret) {
+            error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+                         "0x%"HWADDR_PRIx", %p) = %d (%m)",
+                         container, iotlb->iova,
+                         iotlb->addr_mask + 1, vaddr, ret);
+        }
+    } else {
+        ret = vfio_dma_unmap(container, iotlb->iova, iotlb->addr_mask + 1);
+        if (ret) {
+            error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+                         "0x%"HWADDR_PRIx") = %d (%m)",
+                         container, iotlb->iova,
+                         iotlb->addr_mask + 1, ret);
+        }
+    }
+}
+
 static void vfio_listener_region_add(MemoryListener *listener,
                                      MemoryRegionSection *section)
 {
     VFIOContainer *container = container_of(listener, VFIOContainer,
                                             iommu_data.type1.listener);
     hwaddr iova, end;
+    Int128 llend;
     void *vaddr;
     int ret;
 
-    assert(!memory_region_is_iommu(section->mr));
-
     if (vfio_listener_skipped_section(section)) {
         DPRINTF("SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
                 section->offset_within_address_space,
@@ -2268,21 +2486,65 @@
     }
 
     iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
-    end = (section->offset_within_address_space + int128_get64(section->size)) &
-          TARGET_PAGE_MASK;
+    llend = int128_make64(section->offset_within_address_space);
+    llend = int128_add(llend, section->size);
+    llend = int128_and(llend, int128_exts64(TARGET_PAGE_MASK));
 
-    if (iova >= end) {
+    if (int128_ge(int128_make64(iova), llend)) {
         return;
     }
 
+    memory_region_ref(section->mr);
+
+    if (memory_region_is_iommu(section->mr)) {
+        VFIOGuestIOMMU *giommu;
+
+        DPRINTF("region_add [iommu] %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+                iova, int128_get64(int128_sub(llend, int128_one())));
+        /*
+         * FIXME: We should do some checking to see if the
+         * capabilities of the host VFIO IOMMU are adequate to model
+         * the guest IOMMU
+         *
+         * FIXME: For VFIO iommu types which have KVM acceleration to
+         * avoid bouncing all map/unmaps through qemu this way, this
+         * would be the right place to wire that up (tell the KVM
+         * device emulation the VFIO iommu handles to use).
+         */
+        /*
+         * This assumes that the guest IOMMU is empty of
+         * mappings at this point.
+         *
+         * One way of doing this is:
+         * 1. Avoid sharing IOMMUs between emulated devices or different
+         * IOMMU groups.
+         * 2. Implement VFIO_IOMMU_ENABLE in the host kernel to fail if
+         * there are some mappings in IOMMU.
+         *
+         * VFIO on SPAPR does that. Other IOMMU models may do that different,
+         * they must make sure there are no existing mappings or
+         * loop through existing mappings to map them into VFIO.
+         */
+        giommu = g_malloc0(sizeof(*giommu));
+        giommu->iommu = section->mr;
+        giommu->container = container;
+        giommu->n.notify = vfio_iommu_map_notify;
+        QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
+        memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
+
+        return;
+    }
+
+    /* Here we assume that memory_region_is_ram(section->mr)==true */
+
+    end = int128_get64(llend);
     vaddr = memory_region_get_ram_ptr(section->mr) +
             section->offset_within_region +
             (iova - section->offset_within_address_space);
 
-    DPRINTF("region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
+    DPRINTF("region_add [ram] %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
             iova, end - 1, vaddr);
 
-    memory_region_ref(section->mr);
     ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
     if (ret) {
         error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
@@ -2326,6 +2588,27 @@
         return;
     }
 
+    if (memory_region_is_iommu(section->mr)) {
+        VFIOGuestIOMMU *giommu;
+
+        QLIST_FOREACH(giommu, &container->giommu_list, giommu_next) {
+            if (giommu->iommu == section->mr) {
+                memory_region_unregister_iommu_notifier(&giommu->n);
+                QLIST_REMOVE(giommu, giommu_next);
+                g_free(giommu);
+                break;
+            }
+        }
+
+        /*
+         * FIXME: We assume the one big unmap below is adequate to
+         * remove any individual page mappings in the IOMMU which
+         * might have been copied into VFIO. This works for a page table
+         * based IOMMU where a big unmap flattens a large range of IO-PTEs.
+         * That may not be true for all IOMMU types.
+         */
+    }
+
     iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
     end = (section->offset_within_address_space + int128_get64(section->size)) &
           TARGET_PAGE_MASK;
@@ -3274,16 +3557,43 @@
 #endif
 }
 
-static int vfio_connect_container(VFIOGroup *group)
+static VFIOAddressSpace *vfio_get_address_space(AddressSpace *as)
+{
+    VFIOAddressSpace *space;
+
+    QLIST_FOREACH(space, &vfio_address_spaces, list) {
+        if (space->as == as) {
+            return space;
+        }
+    }
+
+    /* No suitable VFIOAddressSpace, create a new one */
+    space = g_malloc0(sizeof(*space));
+    space->as = as;
+    QLIST_INIT(&space->containers);
+
+    QLIST_INSERT_HEAD(&vfio_address_spaces, space, list);
+
+    return space;
+}
+
+static void vfio_put_address_space(VFIOAddressSpace *space)
+{
+    if (QLIST_EMPTY(&space->containers)) {
+        QLIST_REMOVE(space, list);
+        g_free(space);
+    }
+}
+
+static int vfio_connect_container(VFIOGroup *group, AddressSpace *as)
 {
     VFIOContainer *container;
     int ret, fd;
+    VFIOAddressSpace *space;
 
-    if (group->container) {
-        return 0;
-    }
+    space = vfio_get_address_space(as);
 
-    QLIST_FOREACH(container, &container_list, next) {
+    QLIST_FOREACH(container, &space->containers, next) {
         if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
             group->container = container;
             QLIST_INSERT_HEAD(&container->group_list, group, container_next);
@@ -3294,35 +3604,35 @@
     fd = qemu_open("/dev/vfio/vfio", O_RDWR);
     if (fd < 0) {
         error_report("vfio: failed to open /dev/vfio/vfio: %m");
-        return -errno;
+        ret = -errno;
+        goto put_space_exit;
     }
 
     ret = ioctl(fd, VFIO_GET_API_VERSION);
     if (ret != VFIO_API_VERSION) {
         error_report("vfio: supported vfio version: %d, "
                      "reported version: %d", VFIO_API_VERSION, ret);
-        close(fd);
-        return -EINVAL;
+        ret = -EINVAL;
+        goto close_fd_exit;
     }
 
     container = g_malloc0(sizeof(*container));
+    container->space = space;
     container->fd = fd;
 
     if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
         ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
         if (ret) {
             error_report("vfio: failed to set group container: %m");
-            g_free(container);
-            close(fd);
-            return -errno;
+            ret = -errno;
+            goto free_container_exit;
         }
 
         ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
         if (ret) {
             error_report("vfio: failed to set iommu for container: %m");
-            g_free(container);
-            close(fd);
-            return -errno;
+            ret = -errno;
+            goto free_container_exit;
         }
 
         container->iommu_data.type1.listener = vfio_memory_listener;
@@ -3333,29 +3643,39 @@
 
         if (container->iommu_data.type1.error) {
             ret = container->iommu_data.type1.error;
-            vfio_listener_release(container);
-            g_free(container);
-            close(fd);
             error_report("vfio: memory listener initialization failed for container");
-            return ret;
+            goto listener_release_exit;
         }
 
         container->iommu_data.type1.initialized = true;
 
     } else {
         error_report("vfio: No available IOMMU models");
-        g_free(container);
-        close(fd);
-        return -EINVAL;
+        ret = -EINVAL;
+        goto free_container_exit;
     }
 
     QLIST_INIT(&container->group_list);
-    QLIST_INSERT_HEAD(&container_list, container, next);
+    QLIST_INSERT_HEAD(&space->containers, container, next);
 
     group->container = container;
     QLIST_INSERT_HEAD(&container->group_list, group, container_next);
 
     return 0;
+
+listener_release_exit:
+    vfio_listener_release(container);
+
+free_container_exit:
+    g_free(container);
+
+close_fd_exit:
+    close(fd);
+
+put_space_exit:
+    vfio_put_address_space(space);
+
+    return ret;
 }
 
 static void vfio_disconnect_container(VFIOGroup *group)
@@ -3371,6 +3691,8 @@
     group->container = NULL;
 
     if (QLIST_EMPTY(&container->group_list)) {
+        VFIOAddressSpace *space = container->space;
+
         if (container->iommu_data.release) {
             container->iommu_data.release(container);
         }
@@ -3378,10 +3700,12 @@
         DPRINTF("vfio_disconnect_container: close container->fd\n");
         close(container->fd);
         g_free(container);
+
+        vfio_put_address_space(space);
     }
 }
 
-static VFIOGroup *vfio_get_group(int groupid)
+static VFIOGroup *vfio_get_group(int groupid, AddressSpace *as)
 {
     VFIOGroup *group;
     char path[32];
@@ -3389,7 +3713,14 @@
 
     QLIST_FOREACH(group, &group_list, next) {
         if (group->groupid == groupid) {
-            return group;
+            /* Found it.  Now is it already in the right context? */
+            if (group->container->space->as == as) {
+                return group;
+            } else {
+                error_report("vfio: group %d used in multiple address spaces",
+                             group->groupid);
+                return NULL;
+            }
         }
     }
 
@@ -3399,34 +3730,27 @@
     group->fd = qemu_open(path, O_RDWR);
     if (group->fd < 0) {
         error_report("vfio: error opening %s: %m", path);
-        g_free(group);
-        return NULL;
+        goto free_group_exit;
     }
 
     if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
         error_report("vfio: error getting group status: %m");
-        close(group->fd);
-        g_free(group);
-        return NULL;
+        goto close_fd_exit;
     }
 
     if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
         error_report("vfio: error, group %d is not viable, please ensure "
                      "all devices within the iommu_group are bound to their "
                      "vfio bus driver.", groupid);
-        close(group->fd);
-        g_free(group);
-        return NULL;
+        goto close_fd_exit;
     }
 
     group->groupid = groupid;
     QLIST_INIT(&group->device_list);
 
-    if (vfio_connect_container(group)) {
+    if (vfio_connect_container(group, as)) {
         error_report("vfio: failed to setup container for group %d", groupid);
-        close(group->fd);
-        g_free(group);
-        return NULL;
+        goto close_fd_exit;
     }
 
     if (QLIST_EMPTY(&group_list)) {
@@ -3438,6 +3762,14 @@
     vfio_kvm_device_add_group(group);
 
     return group;
+
+close_fd_exit:
+    close(group->fd);
+
+free_group_exit:
+    g_free(group);
+
+    return NULL;
 }
 
 static void vfio_put_group(VFIOGroup *group)
@@ -3768,7 +4100,7 @@
     DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
             vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
 
-    group = vfio_get_group(groupid);
+    group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev));
     if (!group) {
         error_report("vfio: failed to get group %d", groupid);
         return -ENOENT;
diff --git a/hw/moxie/moxiesim.c b/hw/moxie/moxiesim.c
index a87ca6d..f4357cf 100644
--- a/hw/moxie/moxiesim.c
+++ b/hw/moxie/moxiesim.c
@@ -107,14 +107,14 @@
     return dev;
 }
 
-static void moxiesim_init(QEMUMachineInitArgs *args)
+static void moxiesim_init(MachineState *machine)
 {
     MoxieCPU *cpu = NULL;
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     CPUMoxieState *env;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/net/cadence_gem.c b/hw/net/cadence_gem.c
index 47e7038..a26861e 100644
--- a/hw/net/cadence_gem.c
+++ b/hw/net/cadence_gem.c
@@ -1,5 +1,5 @@
 /*
- * QEMU Xilinx GEM emulation
+ * QEMU Cadence GEM emulation
  *
  * Copyright (c) 2011 Xilinx, Inc.
  *
diff --git a/hw/openrisc/openrisc_sim.c b/hw/openrisc/openrisc_sim.c
index 8e1af8b..b2b4f9b 100644
--- a/hw/openrisc/openrisc_sim.c
+++ b/hw/openrisc/openrisc_sim.c
@@ -90,11 +90,11 @@
     }
 }
 
-static void openrisc_sim_init(QEMUMachineInitArgs *args)
+static void openrisc_sim_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     OpenRISCCPU *cpu = NULL;
     MemoryRegion *ram;
     int n;
diff --git a/hw/pci/pci.c b/hw/pci/pci.c
index 22fe5ee..8d6a8d4 100644
--- a/hw/pci/pci.c
+++ b/hw/pci/pci.c
@@ -605,13 +605,13 @@
     int dom, bus;
     unsigned slot;
 
-    assert(!root->parent_dev);
-
     if (!root) {
         fprintf(stderr, "No primary PCI bus\n");
         return NULL;
     }
 
+    assert(!root->parent_dev);
+
     if (!devaddr) {
         *devfnp = -1;
         return pci_find_bus_nr(root, 0);
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index f984b3e..223bab9 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -123,7 +123,7 @@
     }
 }
 
-static int ppce500_load_device_tree(QEMUMachineInitArgs *args,
+static int ppce500_load_device_tree(MachineState *machine,
                                     PPCE500Params *params,
                                     hwaddr addr,
                                     hwaddr initrd_base,
@@ -132,7 +132,7 @@
 {
     CPUPPCState *env = first_cpu->env_ptr;
     int ret = -1;
-    uint64_t mem_reg_property[] = { 0, cpu_to_be64(args->ram_size) };
+    uint64_t mem_reg_property[] = { 0, cpu_to_be64(machine->ram_size) };
     int fdt_size;
     void *fdt;
     uint8_t hypercall[16];
@@ -207,7 +207,7 @@
     }
 
     ret = qemu_fdt_setprop_string(fdt, "/chosen", "bootargs",
-                                      args->kernel_cmdline);
+                                      machine->kernel_cmdline);
     if (ret < 0)
         fprintf(stderr, "couldn't set /chosen/bootargs\n");
 
@@ -387,7 +387,7 @@
 }
 
 typedef struct DeviceTreeParams {
-    QEMUMachineInitArgs args;
+    MachineState *machine;
     PPCE500Params params;
     hwaddr addr;
     hwaddr initrd_base;
@@ -397,18 +397,18 @@
 static void ppce500_reset_device_tree(void *opaque)
 {
     DeviceTreeParams *p = opaque;
-    ppce500_load_device_tree(&p->args, &p->params, p->addr, p->initrd_base,
+    ppce500_load_device_tree(p->machine, &p->params, p->addr, p->initrd_base,
                              p->initrd_size, false);
 }
 
-static int ppce500_prep_device_tree(QEMUMachineInitArgs *args,
+static int ppce500_prep_device_tree(MachineState *machine,
                                     PPCE500Params *params,
                                     hwaddr addr,
                                     hwaddr initrd_base,
                                     hwaddr initrd_size)
 {
     DeviceTreeParams *p = g_new(DeviceTreeParams, 1);
-    p->args = *args;
+    p->machine = machine;
     p->params = *params;
     p->addr = addr;
     p->initrd_base = initrd_base;
@@ -417,7 +417,7 @@
     qemu_register_reset(ppce500_reset_device_tree, p);
 
     /* Issue the device tree loader once, so that we get the size of the blob */
-    return ppce500_load_device_tree(args, params, addr, initrd_base,
+    return ppce500_load_device_tree(machine, params, addr, initrd_base,
                                     initrd_size, true);
 }
 
@@ -597,7 +597,7 @@
     return mpic;
 }
 
-void ppce500_init(QEMUMachineInitArgs *args, PPCE500Params *params)
+void ppce500_init(MachineState *machine, PPCE500Params *params)
 {
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -622,8 +622,8 @@
     PPCE500CCSRState *ccsr;
 
     /* Setup CPUs */
-    if (args->cpu_model == NULL) {
-        args->cpu_model = "e500v2_v30";
+    if (machine->cpu_model == NULL) {
+        machine->cpu_model = "e500v2_v30";
     }
 
     irqs = g_malloc0(smp_cpus * sizeof(qemu_irq *));
@@ -633,7 +633,7 @@
         CPUState *cs;
         qemu_irq *input;
 
-        cpu = cpu_ppc_init(args->cpu_model);
+        cpu = cpu_ppc_init(machine->cpu_model);
         if (cpu == NULL) {
             fprintf(stderr, "Unable to initialize CPU!\n");
             exit(1);
@@ -672,7 +672,7 @@
 
     /* Fixup Memory size on a alignment boundary */
     ram_size &= ~(RAM_SIZES_ALIGN - 1);
-    args->ram_size = ram_size;
+    machine->ram_size = ram_size;
 
     /* Register Memory */
     memory_region_init_ram(ram, NULL, "mpc8544ds.ram", ram_size);
@@ -739,11 +739,11 @@
     sysbus_create_simple("e500-spin", MPC8544_SPIN_BASE, NULL);
 
     /* Load kernel. */
-    if (args->kernel_filename) {
-        kernel_size = load_uimage(args->kernel_filename, &entry,
+    if (machine->kernel_filename) {
+        kernel_size = load_uimage(machine->kernel_filename, &entry,
                                   &loadaddr, NULL);
         if (kernel_size < 0) {
-            kernel_size = load_elf(args->kernel_filename, NULL, NULL,
+            kernel_size = load_elf(machine->kernel_filename, NULL, NULL,
                                    &elf_entry, &elf_lowaddr, NULL, 1,
                                    ELF_MACHINE, 0);
             entry = elf_entry;
@@ -752,7 +752,7 @@
         /* XXX try again as binary */
         if (kernel_size < 0) {
             fprintf(stderr, "qemu: could not load kernel '%s'\n",
-                    args->kernel_filename);
+                    machine->kernel_filename);
             exit(1);
         }
 
@@ -764,14 +764,14 @@
     }
 
     /* Load initrd. */
-    if (args->initrd_filename) {
+    if (machine->initrd_filename) {
         initrd_base = (cur_base + INITRD_LOAD_PAD) & ~INITRD_PAD_MASK;
-        initrd_size = load_image_targphys(args->initrd_filename, initrd_base,
+        initrd_size = load_image_targphys(machine->initrd_filename, initrd_base,
                                           ram_size - initrd_base);
 
         if (initrd_size < 0) {
             fprintf(stderr, "qemu: could not load initial ram disk '%s'\n",
-                    args->initrd_filename);
+                    machine->initrd_filename);
             exit(1);
         }
 
@@ -779,11 +779,11 @@
     }
 
     /* If we're loading a kernel directly, we must load the device tree too. */
-    if (args->kernel_filename) {
+    if (machine->kernel_filename) {
         struct boot_info *boot_info;
         int dt_size;
 
-        dt_size = ppce500_prep_device_tree(args, params, dt_base,
+        dt_size = ppce500_prep_device_tree(machine, params, dt_base,
                                            initrd_base, initrd_size);
         if (dt_size < 0) {
             fprintf(stderr, "couldn't load device tree\n");
diff --git a/hw/ppc/e500.h b/hw/ppc/e500.h
index 52726a2..08b25fa 100644
--- a/hw/ppc/e500.h
+++ b/hw/ppc/e500.h
@@ -13,6 +13,6 @@
     int mpic_version;
 } PPCE500Params;
 
-void ppce500_init(QEMUMachineInitArgs *args, PPCE500Params *params);
+void ppce500_init(MachineState *machine, PPCE500Params *params);
 
 #endif
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 7d5357e..27df31d 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -28,7 +28,7 @@
                      sizeof(compatible));
 }
 
-static void e500plat_init(QEMUMachineInitArgs *args)
+static void e500plat_init(MachineState *machine)
 {
     PPCE500Params params = {
         .pci_first_slot = 0x1,
@@ -43,7 +43,7 @@
         params.mpic_version = OPENPIC_MODEL_FSL_MPIC_20;
     }
 
-    ppce500_init(args, &params);
+    ppce500_init(machine, &params);
 }
 
 static QEMUMachine e500plat_machine = {
diff --git a/hw/ppc/mac_newworld.c b/hw/ppc/mac_newworld.c
index 5e79575..4bdaa8d 100644
--- a/hw/ppc/mac_newworld.c
+++ b/hw/ppc/mac_newworld.c
@@ -140,14 +140,14 @@
 }
 
 /* PowerPC Mac99 hardware initialisation */
-static void ppc_core99_init(QEMUMachineInitArgs *args)
+static void ppc_core99_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
-    const char *boot_device = args->boot_order;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *boot_device = machine->boot_order;
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
     char *filename;
diff --git a/hw/ppc/mac_oldworld.c b/hw/ppc/mac_oldworld.c
index 2f27754..77598e4 100644
--- a/hw/ppc/mac_oldworld.c
+++ b/hw/ppc/mac_oldworld.c
@@ -71,14 +71,14 @@
     cpu_reset(CPU(cpu));
 }
 
-static void ppc_heathrow_init(QEMUMachineInitArgs *args)
+static void ppc_heathrow_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
-    const char *boot_device = args->boot_order;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *boot_device = machine->boot_order;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index 292c709..b99f74a 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -26,7 +26,7 @@
                      sizeof(compatible));
 }
 
-static void mpc8544ds_init(QEMUMachineInitArgs *args)
+static void mpc8544ds_init(MachineState *machine)
 {
     PPCE500Params params = {
         .pci_first_slot = 0x11,
@@ -35,7 +35,7 @@
         .mpic_version = OPENPIC_MODEL_FSL_MPIC_20,
     };
 
-    ppce500_init(args, &params);
+    ppce500_init(machine, &params);
 }
 
 
diff --git a/hw/ppc/ppc405_boards.c b/hw/ppc/ppc405_boards.c
index f1a8f67..98ad2d7 100644
--- a/hw/ppc/ppc405_boards.c
+++ b/hw/ppc/ppc405_boards.c
@@ -172,12 +172,12 @@
     qemu_register_reset(&ref405ep_fpga_reset, fpga);
 }
 
-static void ref405ep_init(QEMUMachineInitArgs *args)
+static void ref405ep_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     ppc4xx_bd_info_t bd;
     CPUPPCState *env;
@@ -499,11 +499,11 @@
     qemu_register_reset(&taihu_cpld_reset, cpld);
 }
 
-static void taihu_405ep_init(QEMUMachineInitArgs *args)
+static void taihu_405ep_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *kernel_filename = args->kernel_filename;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *initrd_filename = machine->initrd_filename;
     char *filename;
     qemu_irq *pic;
     MemoryRegion *sysmem = get_system_memory();
diff --git a/hw/ppc/ppc440_bamboo.c b/hw/ppc/ppc440_bamboo.c
index 2ddc2ed..81a06d3 100644
--- a/hw/ppc/ppc440_bamboo.c
+++ b/hw/ppc/ppc440_bamboo.c
@@ -156,13 +156,13 @@
     mmubooke_create_initial_mapping(env, 0, 0);
 }
 
-static void bamboo_init(QEMUMachineInitArgs *args)
+static void bamboo_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     unsigned int pci_irq_nrs[4] = { 28, 27, 26, 25 };
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *isa = g_new(MemoryRegion, 1);
diff --git a/hw/ppc/prep.c b/hw/ppc/prep.c
index 5859373..2383254 100644
--- a/hw/ppc/prep.c
+++ b/hw/ppc/prep.c
@@ -364,14 +364,14 @@
 static PortioList prep_port_list;
 
 /* PowerPC PREP hardware initialisation */
-static void ppc_prep_init(QEMUMachineInitArgs *args)
+static void ppc_prep_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
-    const char *boot_device = args->boot_order;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *boot_device = machine->boot_order;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
diff --git a/hw/ppc/spapr.c b/hw/ppc/spapr.c
index b4ce950..57e9578 100644
--- a/hw/ppc/spapr.c
+++ b/hw/ppc/spapr.c
@@ -1140,14 +1140,14 @@
 };
 
 /* pSeries LPAR / sPAPR hardware init */
-static void ppc_spapr_init(QEMUMachineInitArgs *args)
+static void ppc_spapr_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
-    const char *boot_device = args->boot_order;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
+    const char *boot_device = machine->boot_order;
     PowerPCCPU *cpu;
     CPUPPCState *env;
     PCIHostState *phb;
diff --git a/hw/ppc/virtex_ml507.c b/hw/ppc/virtex_ml507.c
index 3e3569d..02b4f82 100644
--- a/hw/ppc/virtex_ml507.c
+++ b/hw/ppc/virtex_ml507.c
@@ -194,12 +194,12 @@
     return fdt_size;
 }
 
-static void virtex_init(QEMUMachineInitArgs *args)
+static void virtex_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
     hwaddr initrd_base = 0;
     int initrd_size = 0;
     MemoryRegion *address_space_mem = get_system_memory();
@@ -275,14 +275,14 @@
         boot_info.ima_size = kernel_size;
 
         /* Load initrd. */
-        if (args->initrd_filename) {
+        if (machine->initrd_filename) {
             initrd_base = high = ROUND_UP(high, 4);
-            initrd_size = load_image_targphys(args->initrd_filename,
+            initrd_size = load_image_targphys(machine->initrd_filename,
                                               high, ram_size - high);
 
             if (initrd_size < 0) {
                 error_report("couldn't load ram disk '%s'",
-                             args->initrd_filename);
+                             machine->initrd_filename);
                 exit(1);
             }
             high = ROUND_UP(high + initrd_size, 4);
diff --git a/hw/s390x/css.c b/hw/s390x/css.c
index 122cc7e..2678e44 100644
--- a/hw/s390x/css.c
+++ b/hw/s390x/css.c
@@ -16,6 +16,7 @@
 #include "ioinst.h"
 #include "css.h"
 #include "trace.h"
+#include "hw/s390x/s390_flic.h"
 
 typedef struct CrwContainer {
     CRW crw;
@@ -39,6 +40,13 @@
     ChpInfo chpids[MAX_CHPID + 1];
 } CssImage;
 
+typedef struct IoAdapter {
+    uint32_t id;
+    uint8_t type;
+    uint8_t isc;
+    QTAILQ_ENTRY(IoAdapter) sibling;
+} IoAdapter;
+
 typedef struct ChannelSubSys {
     QTAILQ_HEAD(, CrwContainer) pending_crws;
     bool do_crw_mchk;
@@ -49,6 +57,7 @@
     uint64_t chnmon_area;
     CssImage *css[MAX_CSSID + 1];
     uint8_t default_cssid;
+    QTAILQ_HEAD(, IoAdapter) io_adapters;
 } ChannelSubSys;
 
 static ChannelSubSys *channel_subsys;
@@ -69,6 +78,46 @@
     return 0;
 }
 
+int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
+                            bool maskable, uint32_t *id)
+{
+    IoAdapter *adapter;
+    bool found = false;
+    int ret;
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    *id = 0;
+    QTAILQ_FOREACH(adapter, &channel_subsys->io_adapters, sibling) {
+        if ((adapter->type == type) && (adapter->isc == isc)) {
+            *id = adapter->id;
+            found = true;
+            ret = 0;
+            break;
+        }
+        if (adapter->id >= *id) {
+            *id = adapter->id + 1;
+        }
+    }
+    if (found) {
+        goto out;
+    }
+    adapter = g_new0(IoAdapter, 1);
+    ret = fsc->register_io_adapter(fs, *id, isc, swap, maskable);
+    if (ret == 0) {
+        adapter->id = *id;
+        adapter->isc = isc;
+        adapter->type = type;
+        QTAILQ_INSERT_TAIL(&channel_subsys->io_adapters, adapter, sibling);
+    } else {
+        g_free(adapter);
+        fprintf(stderr, "Unexpected error %d when registering adapter %d\n",
+                ret, *id);
+    }
+out:
+    return ret;
+}
+
 uint16_t css_build_subchannel_id(SubchDev *sch)
 {
     if (channel_subsys->max_cssid > 0) {
@@ -1235,6 +1284,7 @@
     channel_subsys->do_crw_mchk = true;
     channel_subsys->crws_lost = false;
     channel_subsys->chnmon_active = false;
+    QTAILQ_INIT(&channel_subsys->io_adapters);
 }
 machine_init(css_init);
 
diff --git a/hw/s390x/css.h b/hw/s390x/css.h
index 220169e..6586106 100644
--- a/hw/s390x/css.h
+++ b/hw/s390x/css.h
@@ -98,4 +98,8 @@
                            int hotplugged, int add);
 void css_generate_chp_crws(uint8_t cssid, uint8_t chpid);
 void css_adapter_interrupt(uint8_t isc);
+
+#define CSS_IO_ADAPTER_VIRTIO 1
+int css_register_io_adapter(uint8_t type, uint8_t isc, bool swap,
+                            bool maskable, uint32_t *id);
 #endif
diff --git a/hw/s390x/s390-virtio-ccw.c b/hw/s390x/s390-virtio-ccw.c
index 0d4f6ae..42f5cec 100644
--- a/hw/s390x/s390-virtio-ccw.c
+++ b/hw/s390x/s390-virtio-ccw.c
@@ -79,9 +79,9 @@
                                    virtio_ccw_hcall_early_printk);
 }
 
-static void ccw_init(QEMUMachineInitArgs *args)
+static void ccw_init(MachineState *machine)
 {
-    ram_addr_t my_ram_size = args->ram_size;
+    ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     int shift = 0;
@@ -102,8 +102,8 @@
     /* get a BUS */
     css_bus = virtual_css_bus_init();
     s390_sclp_init();
-    s390_init_ipl_dev(args->kernel_filename, args->kernel_cmdline,
-                      args->initrd_filename, "s390-ccw.img");
+    s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
+                      machine->initrd_filename, "s390-ccw.img");
     s390_flic_init();
 
     /* register hypercalls */
@@ -118,7 +118,7 @@
     storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
 
     /* init CPUs */
-    s390_init_cpus(args->cpu_model, storage_keys);
+    s390_init_cpus(machine->cpu_model, storage_keys);
 
     if (kvm_enabled()) {
         kvm_s390_enable_css_support(s390_cpu_addr2state(0));
diff --git a/hw/s390x/s390-virtio.c b/hw/s390x/s390-virtio.c
index aef2003..93c7ace 100644
--- a/hw/s390x/s390-virtio.c
+++ b/hw/s390x/s390-virtio.c
@@ -224,9 +224,9 @@
 }
 
 /* PC hardware initialisation */
-static void s390_init(QEMUMachineInitArgs *args)
+static void s390_init(MachineState *machine)
 {
-    ram_addr_t my_ram_size = args->ram_size;
+    ram_addr_t my_ram_size = machine->ram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     int shift = 0;
@@ -248,8 +248,8 @@
     /* get a BUS */
     s390_bus = s390_virtio_bus_init(&my_ram_size);
     s390_sclp_init();
-    s390_init_ipl_dev(args->kernel_filename, args->kernel_cmdline,
-                      args->initrd_filename, ZIPL_FILENAME);
+    s390_init_ipl_dev(machine->kernel_filename, machine->kernel_cmdline,
+                      machine->initrd_filename, ZIPL_FILENAME);
     s390_flic_init();
 
     /* register hypercalls */
@@ -273,7 +273,7 @@
     storage_keys = g_malloc0(my_ram_size / TARGET_PAGE_SIZE);
 
     /* init CPUs */
-    s390_init_cpus(args->cpu_model, storage_keys);
+    s390_init_cpus(machine->cpu_model, storage_keys);
 
     /* Create VirtIO network adapters */
     s390_create_virtio_net((BusState *)s390_bus, "virtio-net-s390");
diff --git a/hw/s390x/virtio-ccw.c b/hw/s390x/virtio-ccw.c
index 1cb4e2c..c4f21d3 100644
--- a/hw/s390x/virtio-ccw.c
+++ b/hw/s390x/virtio-ccw.c
@@ -21,12 +21,77 @@
 #include "hw/sysbus.h"
 #include "qemu/bitops.h"
 #include "hw/virtio/virtio-bus.h"
+#include "hw/s390x/adapter.h"
+#include "hw/s390x/s390_flic.h"
 
 #include "ioinst.h"
 #include "css.h"
 #include "virtio-ccw.h"
 #include "trace.h"
 
+static QTAILQ_HEAD(, IndAddr) indicator_addresses =
+    QTAILQ_HEAD_INITIALIZER(indicator_addresses);
+
+static IndAddr *get_indicator(hwaddr ind_addr, int len)
+{
+    IndAddr *indicator;
+
+    QTAILQ_FOREACH(indicator, &indicator_addresses, sibling) {
+        if (indicator->addr == ind_addr) {
+            indicator->refcnt++;
+            return indicator;
+        }
+    }
+    indicator = g_new0(IndAddr, 1);
+    indicator->addr = ind_addr;
+    indicator->len = len;
+    indicator->refcnt = 1;
+    QTAILQ_INSERT_TAIL(&indicator_addresses, indicator, sibling);
+    return indicator;
+}
+
+static int s390_io_adapter_map(AdapterInfo *adapter, uint64_t map_addr,
+                               bool do_map)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    return fsc->io_adapter_map(fs, adapter->adapter_id, map_addr, do_map);
+}
+
+static void release_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    assert(indicator->refcnt > 0);
+    indicator->refcnt--;
+    if (indicator->refcnt > 0) {
+        return;
+    }
+    QTAILQ_REMOVE(&indicator_addresses, indicator, sibling);
+    if (indicator->map) {
+        s390_io_adapter_map(adapter, indicator->map, false);
+    }
+    g_free(indicator);
+}
+
+static int map_indicator(AdapterInfo *adapter, IndAddr *indicator)
+{
+    int ret;
+
+    if (indicator->map) {
+        return 0; /* already mapped is not an error */
+    }
+    indicator->map = indicator->addr;
+    ret = s390_io_adapter_map(adapter, indicator->map, true);
+    if ((ret != 0) && (ret != -ENOSYS)) {
+        goto out_err;
+    }
+    return 0;
+
+out_err:
+    indicator->map = 0;
+    return ret;
+}
+
 static void virtio_ccw_bus_new(VirtioBusState *bus, size_t bus_size,
                                VirtioCcwDevice *dev);
 
@@ -445,7 +510,7 @@
             ret = -EFAULT;
         } else {
             indicators = ldq_phys(&address_space_memory, ccw.cda);
-            dev->indicators = indicators;
+            dev->indicators = get_indicator(indicators, sizeof(uint64_t));
             sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
             ret = 0;
         }
@@ -465,7 +530,7 @@
             ret = -EFAULT;
         } else {
             indicators = ldq_phys(&address_space_memory, ccw.cda);
-            dev->indicators2 = indicators;
+            dev->indicators2 = get_indicator(indicators, sizeof(uint64_t));
             sch->curr_status.scsw.count = ccw.count - sizeof(indicators);
             ret = 0;
         }
@@ -517,13 +582,20 @@
                 ret = -EFAULT;
             } else {
                 len = hw_len;
-                dev->summary_indicator = thinint->summary_indicator;
-                dev->indicators = thinint->device_indicator;
+                dev->summary_indicator =
+                    get_indicator(thinint->summary_indicator, sizeof(uint8_t));
+                dev->indicators = get_indicator(thinint->device_indicator,
+                                                thinint->ind_bit / 8 + 1);
                 dev->thinint_isc = thinint->isc;
-                dev->ind_bit = thinint->ind_bit;
+                dev->routes.adapter.ind_offset = thinint->ind_bit;
+                dev->routes.adapter.summary_offset = 7;
                 cpu_physical_memory_unmap(thinint, hw_len, 0, hw_len);
-                sch->thinint_active = ((dev->indicators != 0) &&
-                                       (dev->summary_indicator != 0));
+                ret = css_register_io_adapter(CSS_IO_ADAPTER_VIRTIO,
+                                              dev->thinint_isc, true, false,
+                                              &dev->routes.adapter.adapter_id);
+                assert(ret == 0);
+                sch->thinint_active = ((dev->indicators != NULL) &&
+                                       (dev->summary_indicator != NULL));
                 sch->curr_status.scsw.count = ccw.count - len;
                 ret = 0;
             }
@@ -554,7 +626,7 @@
     sch->driver_data = dev;
     dev->sch = sch;
 
-    dev->indicators = 0;
+    dev->indicators = NULL;
 
     /* Initialize subchannel structure. */
     sch->channel_prog = 0x0;
@@ -693,7 +765,10 @@
         css_subch_assign(sch->cssid, sch->ssid, sch->schid, sch->devno, NULL);
         g_free(sch);
     }
-    dev->indicators = 0;
+    if (dev->indicators) {
+        release_indicator(&dev->routes.adapter, dev->indicators);
+        dev->indicators = NULL;
+    }
     return 0;
 }
 
@@ -950,17 +1025,19 @@
              * ind_bit indicates the start of the indicators in a big
              * endian notation.
              */
-            virtio_set_ind_atomic(sch, dev->indicators +
-                                  (dev->ind_bit + vector) / 8,
-                                  0x80 >> ((dev->ind_bit + vector) % 8));
-            if (!virtio_set_ind_atomic(sch, dev->summary_indicator,
+            uint64_t ind_bit = dev->routes.adapter.ind_offset;
+
+            virtio_set_ind_atomic(sch, dev->indicators->addr +
+                                  (ind_bit + vector) / 8,
+                                  0x80 >> ((ind_bit + vector) % 8));
+            if (!virtio_set_ind_atomic(sch, dev->summary_indicator->addr,
                                        0x01)) {
                 css_adapter_interrupt(dev->thinint_isc);
             }
         } else {
-            indicators = ldq_phys(&address_space_memory, dev->indicators);
+            indicators = ldq_phys(&address_space_memory, dev->indicators->addr);
             indicators |= 1ULL << vector;
-            stq_phys(&address_space_memory, dev->indicators, indicators);
+            stq_phys(&address_space_memory, dev->indicators->addr, indicators);
             css_conditional_io_interrupt(sch);
         }
     } else {
@@ -968,9 +1045,9 @@
             return;
         }
         vector = 0;
-        indicators = ldq_phys(&address_space_memory, dev->indicators2);
+        indicators = ldq_phys(&address_space_memory, dev->indicators2->addr);
         indicators |= 1ULL << vector;
-        stq_phys(&address_space_memory, dev->indicators2, indicators);
+        stq_phys(&address_space_memory, dev->indicators2->addr, indicators);
         css_conditional_io_interrupt(sch);
     }
 }
@@ -991,9 +1068,18 @@
     virtio_ccw_stop_ioeventfd(dev);
     virtio_reset(vdev);
     css_reset_sch(dev->sch);
-    dev->indicators = 0;
-    dev->indicators2 = 0;
-    dev->summary_indicator = 0;
+    if (dev->indicators) {
+        release_indicator(&dev->routes.adapter, dev->indicators);
+        dev->indicators = NULL;
+    }
+    if (dev->indicators2) {
+        release_indicator(&dev->routes.adapter, dev->indicators2);
+        dev->indicators2 = NULL;
+    }
+    if (dev->summary_indicator) {
+        release_indicator(&dev->routes.adapter, dev->summary_indicator);
+        dev->summary_indicator = NULL;
+    }
 }
 
 static void virtio_ccw_vmstate_change(DeviceState *d, bool running)
@@ -1027,6 +1113,79 @@
     return virtio_ccw_set_guest2host_notifier(dev, n, assign, false);
 }
 
+static int virtio_ccw_get_mappings(VirtioCcwDevice *dev)
+{
+    int r;
+
+    if (!dev->sch->thinint_active) {
+        return -EINVAL;
+    }
+
+    r = map_indicator(&dev->routes.adapter, dev->summary_indicator);
+    if (r) {
+        return r;
+    }
+    r = map_indicator(&dev->routes.adapter, dev->indicators);
+    if (r) {
+        return r;
+    }
+    dev->routes.adapter.summary_addr = dev->summary_indicator->map;
+    dev->routes.adapter.ind_addr = dev->indicators->map;
+
+    return 0;
+}
+
+static int virtio_ccw_setup_irqroutes(VirtioCcwDevice *dev, int nvqs)
+{
+    int i;
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    int ret;
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    ret = virtio_ccw_get_mappings(dev);
+    if (ret) {
+        return ret;
+    }
+    for (i = 0; i < nvqs; i++) {
+        if (!virtio_queue_get_num(vdev, i)) {
+            break;
+        }
+    }
+    dev->routes.num_routes = i;
+    return fsc->add_adapter_routes(fs, &dev->routes);
+}
+
+static void virtio_ccw_release_irqroutes(VirtioCcwDevice *dev, int nvqs)
+{
+    S390FLICState *fs = s390_get_flic();
+    S390FLICStateClass *fsc = S390_FLIC_COMMON_GET_CLASS(fs);
+
+    fsc->release_adapter_routes(fs, &dev->routes);
+}
+
+static int virtio_ccw_add_irqfd(VirtioCcwDevice *dev, int n)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
+
+    return kvm_irqchip_add_irqfd_notifier(kvm_state, notifier, NULL,
+                                          dev->routes.gsi[n]);
+}
+
+static void virtio_ccw_remove_irqfd(VirtioCcwDevice *dev, int n)
+{
+    VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    VirtQueue *vq = virtio_get_queue(vdev, n);
+    EventNotifier *notifier = virtio_queue_get_guest_notifier(vq);
+    int ret;
+
+    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, notifier,
+                                            dev->routes.gsi[n]);
+    assert(ret == 0);
+}
+
 static int virtio_ccw_set_guest_notifier(VirtioCcwDevice *dev, int n,
                                          bool assign, bool with_irqfd)
 {
@@ -1042,11 +1201,17 @@
             return r;
         }
         virtio_queue_set_guest_notifier_fd_handler(vq, true, with_irqfd);
-        /* We do not support irqfd for classic I/O interrupts, because the
-         * classic interrupts are intermixed with the subchannel status, that
-         * is queried with test subchannel. We want to use vhost, though.
-         * Lets make sure to have vhost running and wire up the irq fd to
-         * land in qemu (and only the irq fd) in this code.
+        if (with_irqfd) {
+            r = virtio_ccw_add_irqfd(dev, n);
+            if (r) {
+                virtio_queue_set_guest_notifier_fd_handler(vq, false,
+                                                           with_irqfd);
+                return r;
+            }
+        }
+        /*
+         * We do not support individual masking for channel devices, so we
+         * need to manually trigger any guest masking callbacks here.
          */
         if (k->guest_notifier_mask) {
             k->guest_notifier_mask(vdev, n, false);
@@ -1060,6 +1225,9 @@
         if (k->guest_notifier_mask) {
             k->guest_notifier_mask(vdev, n, true);
         }
+        if (with_irqfd) {
+            virtio_ccw_remove_irqfd(dev, n);
+        }
         virtio_queue_set_guest_notifier_fd_handler(vq, false, with_irqfd);
         event_notifier_cleanup(notifier);
     }
@@ -1071,24 +1239,39 @@
 {
     VirtioCcwDevice *dev = VIRTIO_CCW_DEVICE(d);
     VirtIODevice *vdev = virtio_bus_get_device(&dev->bus);
+    bool with_irqfd = dev->sch->thinint_active && kvm_irqfds_enabled();
     int r, n;
 
+    if (with_irqfd && assigned) {
+        /* irq routes need to be set up before assigning irqfds */
+        r = virtio_ccw_setup_irqroutes(dev, nvqs);
+        if (r < 0) {
+            goto irqroute_error;
+        }
+    }
     for (n = 0; n < nvqs; n++) {
         if (!virtio_queue_get_num(vdev, n)) {
             break;
         }
-        /* false -> true, as soon as irqfd works */
-        r = virtio_ccw_set_guest_notifier(dev, n, assigned, false);
+        r = virtio_ccw_set_guest_notifier(dev, n, assigned, with_irqfd);
         if (r < 0) {
             goto assign_error;
         }
     }
+    if (with_irqfd && !assigned) {
+        /* release irq routes after irqfds have been released */
+        virtio_ccw_release_irqroutes(dev, nvqs);
+    }
     return 0;
 
 assign_error:
     while (--n >= 0) {
         virtio_ccw_set_guest_notifier(dev, n, !assigned, false);
     }
+irqroute_error:
+    if (with_irqfd && assigned) {
+        virtio_ccw_release_irqroutes(dev, nvqs);
+    }
     return r;
 }
 
diff --git a/hw/s390x/virtio-ccw.h b/hw/s390x/virtio-ccw.h
index 4393e44..b8b8a8a 100644
--- a/hw/s390x/virtio-ccw.h
+++ b/hw/s390x/virtio-ccw.h
@@ -22,6 +22,7 @@
 #include <hw/virtio/virtio-balloon.h>
 #include <hw/virtio/virtio-rng.h>
 #include <hw/virtio/virtio-bus.h>
+#include <hw/s390x/s390_flic.h>
 
 #define VIRTUAL_CSSID 0xfe
 
@@ -75,6 +76,14 @@
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT 1
 #define VIRTIO_CCW_FLAG_USE_IOEVENTFD   (1 << VIRTIO_CCW_FLAG_USE_IOEVENTFD_BIT)
 
+typedef struct IndAddr {
+    hwaddr addr;
+    uint64_t map;
+    unsigned long refcnt;
+    int len;
+    QTAILQ_ENTRY(IndAddr) sibling;
+} IndAddr;
+
 struct VirtioCcwDevice {
     DeviceState parent_obj;
     SubchDev *sch;
@@ -85,10 +94,11 @@
     bool ioeventfd_disabled;
     uint32_t flags;
     uint8_t thinint_isc;
+    AdapterRoutes routes;
     /* Guest provided values: */
-    hwaddr indicators;
-    hwaddr indicators2;
-    hwaddr summary_indicator;
+    IndAddr *indicators;
+    IndAddr *indicators2;
+    IndAddr *summary_indicator;
     uint64_t ind_bit;
 };
 
diff --git a/hw/scsi/megasas.c b/hw/scsi/megasas.c
index baee46f..0e109a2 100644
--- a/hw/scsi/megasas.c
+++ b/hw/scsi/megasas.c
@@ -728,8 +728,8 @@
     snprintf(info.package_version, 0x60, "%s-QEMU", QEMU_VERSION);
     memcpy(info.image_component[0].name, "APP", 3);
     memcpy(info.image_component[0].version, MEGASAS_VERSION "-QEMU", 9);
-    memcpy(info.image_component[0].build_date, __DATE__, 11);
-    memcpy(info.image_component[0].build_time, __TIME__, 8);
+    memcpy(info.image_component[0].build_date, "Apr  1 2014", 11);
+    memcpy(info.image_component[0].build_time, "12:34:56", 8);
     info.image_component_count = 1;
     if (pci_dev->has_rom) {
         uint8_t biosver[32];
diff --git a/hw/scsi/scsi-bus.c b/hw/scsi/scsi-bus.c
index abe7302..06399fa 100644
--- a/hw/scsi/scsi-bus.c
+++ b/hw/scsi/scsi-bus.c
@@ -938,6 +938,7 @@
         if (cmd->xfer == 0) {
             cmd->xfer = 256;
         }
+        /* fall through */
     case WRITE_10:
     case WRITE_VERIFY_10:
     case WRITE_12:
@@ -952,6 +953,7 @@
         if (cmd->xfer == 0) {
             cmd->xfer = 256;
         }
+        /* fall through */
     case READ_10:
     case RECOVER_BUFFERED_DATA:
     case READ_12:
diff --git a/hw/scsi/vhost-scsi.c b/hw/scsi/vhost-scsi.c
index 3983a5b..668bafa 100644
--- a/hw/scsi/vhost-scsi.c
+++ b/hw/scsi/vhost-scsi.c
@@ -164,8 +164,8 @@
     VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev);
 
-    if ((uint32_t) ldl_raw(&scsiconf->sense_size) != vs->sense_size ||
-        (uint32_t) ldl_raw(&scsiconf->cdb_size) != vs->cdb_size) {
+    if ((uint32_t) ldl_p(&scsiconf->sense_size) != vs->sense_size ||
+        (uint32_t) ldl_p(&scsiconf->cdb_size) != vs->cdb_size) {
         error_report("vhost-scsi does not support changing the sense data and CDB sizes");
         exit(1);
     }
diff --git a/hw/scsi/virtio-scsi.c b/hw/scsi/virtio-scsi.c
index 1752193..b39880a 100644
--- a/hw/scsi/virtio-scsi.c
+++ b/hw/scsi/virtio-scsi.c
@@ -425,16 +425,16 @@
     VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
     VirtIOSCSICommon *s = VIRTIO_SCSI_COMMON(vdev);
 
-    stl_raw(&scsiconf->num_queues, s->conf.num_queues);
-    stl_raw(&scsiconf->seg_max, 128 - 2);
-    stl_raw(&scsiconf->max_sectors, s->conf.max_sectors);
-    stl_raw(&scsiconf->cmd_per_lun, s->conf.cmd_per_lun);
-    stl_raw(&scsiconf->event_info_size, sizeof(VirtIOSCSIEvent));
-    stl_raw(&scsiconf->sense_size, s->sense_size);
-    stl_raw(&scsiconf->cdb_size, s->cdb_size);
-    stw_raw(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
-    stw_raw(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
-    stl_raw(&scsiconf->max_lun, VIRTIO_SCSI_MAX_LUN);
+    stl_p(&scsiconf->num_queues, s->conf.num_queues);
+    stl_p(&scsiconf->seg_max, 128 - 2);
+    stl_p(&scsiconf->max_sectors, s->conf.max_sectors);
+    stl_p(&scsiconf->cmd_per_lun, s->conf.cmd_per_lun);
+    stl_p(&scsiconf->event_info_size, sizeof(VirtIOSCSIEvent));
+    stl_p(&scsiconf->sense_size, s->sense_size);
+    stl_p(&scsiconf->cdb_size, s->cdb_size);
+    stw_p(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
+    stw_p(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
+    stl_p(&scsiconf->max_lun, VIRTIO_SCSI_MAX_LUN);
 }
 
 static void virtio_scsi_set_config(VirtIODevice *vdev,
@@ -443,14 +443,14 @@
     VirtIOSCSIConfig *scsiconf = (VirtIOSCSIConfig *)config;
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(vdev);
 
-    if ((uint32_t) ldl_raw(&scsiconf->sense_size) >= 65536 ||
-        (uint32_t) ldl_raw(&scsiconf->cdb_size) >= 256) {
+    if ((uint32_t) ldl_p(&scsiconf->sense_size) >= 65536 ||
+        (uint32_t) ldl_p(&scsiconf->cdb_size) >= 256) {
         error_report("bad data written to virtio-scsi configuration space");
         exit(1);
     }
 
-    vs->sense_size = ldl_raw(&scsiconf->sense_size);
-    vs->cdb_size = ldl_raw(&scsiconf->cdb_size);
+    vs->sense_size = ldl_p(&scsiconf->sense_size);
+    vs->cdb_size = ldl_p(&scsiconf->cdb_size);
 }
 
 static uint32_t virtio_scsi_get_features(VirtIODevice *vdev,
@@ -498,7 +498,7 @@
                                    uint32_t event, uint32_t reason)
 {
     VirtIOSCSICommon *vs = VIRTIO_SCSI_COMMON(s);
-    VirtIOSCSIReq *req = virtio_scsi_pop_req(s, vs->event_vq);
+    VirtIOSCSIReq *req;
     VirtIOSCSIEvent *evt;
     VirtIODevice *vdev = VIRTIO_DEVICE(s);
     int in_size;
@@ -507,6 +507,7 @@
         return;
     }
 
+    req = virtio_scsi_pop_req(s, vs->event_vq);
     if (!req) {
         s->events_dropped = true;
         return;
diff --git a/hw/sh4/r2d.c b/hw/sh4/r2d.c
index eaeb7ed..95c0246 100644
--- a/hw/sh4/r2d.c
+++ b/hw/sh4/r2d.c
@@ -219,12 +219,12 @@
     char kernel_cmdline[256];
 } boot_params;
 
-static void r2d_init(QEMUMachineInitArgs *args)
+static void r2d_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     SuperHCPU *cpu;
     CPUSH4State *env;
     ResetData *reset_info;
diff --git a/hw/sh4/shix.c b/hw/sh4/shix.c
index 904a966..7c152b4 100644
--- a/hw/sh4/shix.c
+++ b/hw/sh4/shix.c
@@ -39,9 +39,9 @@
 #define BIOS_FILENAME "shix_bios.bin"
 #define BIOS_ADDRESS 0xA0000000
 
-static void shix_init(QEMUMachineInitArgs *args)
+static void shix_init(MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
     int ret;
     SuperHCPU *cpu;
     struct SH7750State *s;
diff --git a/hw/sparc/leon3.c b/hw/sparc/leon3.c
index c16e9e4..827383b 100644
--- a/hw/sparc/leon3.c
+++ b/hw/sparc/leon3.c
@@ -101,11 +101,11 @@
     }
 }
 
-static void leon3_generic_hw_init(QEMUMachineInitArgs *args)
+static void leon3_generic_hw_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     SPARCCPU *cpu;
     CPUSPARCState   *env;
     MemoryRegion *address_space_mem = get_system_memory();
diff --git a/hw/sparc/sun4m.c b/hw/sparc/sun4m.c
index 75adb68..4e793c2 100644
--- a/hw/sparc/sun4m.c
+++ b/hw/sparc/sun4m.c
@@ -867,9 +867,9 @@
 }
 
 static void sun4m_hw_init(const struct sun4m_hwdef *hwdef,
-                          QEMUMachineInitArgs *args)
+                          MachineState *machine)
 {
-    const char *cpu_model = args->cpu_model;
+    const char *cpu_model = machine->cpu_model;
     unsigned int i;
     void *iommu, *espdma, *ledma, *nvram;
     qemu_irq *cpu_irqs[MAX_CPUS], slavio_irq[32], slavio_cpu_irq[MAX_CPUS],
@@ -895,10 +895,10 @@
 
 
     /* set up devices */
-    ram_init(0, args->ram_size, hwdef->max_mem);
+    ram_init(0, machine->ram_size, hwdef->max_mem);
     /* models without ECC don't trap when missing ram is accessed */
     if (!hwdef->ecc_base) {
-        empty_slot_init(args->ram_size, hwdef->max_mem - args->ram_size);
+        empty_slot_init(machine->ram_size, hwdef->max_mem - machine->ram_size);
     }
 
     prom_init(hwdef->slavio_base, bios_name);
@@ -1051,14 +1051,14 @@
         empty_slot_init(hwdef->bpp_base, 0x20);
     }
 
-    kernel_size = sun4m_load_kernel(args->kernel_filename,
-                                    args->initrd_filename,
-                                    args->ram_size);
+    kernel_size = sun4m_load_kernel(machine->kernel_filename,
+                                    machine->initrd_filename,
+                                    machine->ram_size);
 
-    nvram_init(nvram, (uint8_t *)&nd_table[0].macaddr, args->kernel_cmdline,
-               args->boot_order, args->ram_size, kernel_size, graphic_width,
-               graphic_height, graphic_depth, hwdef->nvram_machine_id,
-               "Sun4m");
+    nvram_init(nvram, (uint8_t *)&nd_table[0].macaddr, machine->kernel_cmdline,
+               machine->boot_order, machine->ram_size, kernel_size,
+               graphic_width, graphic_height, graphic_depth,
+               hwdef->nvram_machine_id, "Sun4m");
 
     if (hwdef->ecc_base)
         ecc_init(hwdef->ecc_base, slavio_irq[28],
@@ -1074,20 +1074,20 @@
     fw_cfg_add_i16(fw_cfg, FW_CFG_SUN4M_HEIGHT, graphic_height);
     fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_ADDR, KERNEL_LOAD_ADDR);
     fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_SIZE, kernel_size);
-    if (args->kernel_cmdline) {
+    if (machine->kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, CMDLINE_ADDR);
         pstrcpy_targphys("cmdline", CMDLINE_ADDR, TARGET_PAGE_SIZE,
-                         args->kernel_cmdline);
-        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, args->kernel_cmdline);
+                         machine->kernel_cmdline);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, machine->kernel_cmdline);
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
-                       strlen(args->kernel_cmdline) + 1);
+                       strlen(machine->kernel_cmdline) + 1);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_KERNEL_CMDLINE, 0);
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, 0);
     }
     fw_cfg_add_i32(fw_cfg, FW_CFG_INITRD_ADDR, INITRD_LOAD_ADDR);
     fw_cfg_add_i32(fw_cfg, FW_CFG_INITRD_SIZE, 0); // not used
-    fw_cfg_add_i16(fw_cfg, FW_CFG_BOOT_DEVICE, args->boot_order[0]);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_BOOT_DEVICE, machine->boot_order[0]);
     qemu_register_boot_set(fw_cfg_boot_set, fw_cfg);
 }
 
@@ -1349,57 +1349,57 @@
 };
 
 /* SPARCstation 5 hardware initialisation */
-static void ss5_init(QEMUMachineInitArgs *args)
+static void ss5_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[0], args);
+    sun4m_hw_init(&sun4m_hwdefs[0], machine);
 }
 
 /* SPARCstation 10 hardware initialisation */
-static void ss10_init(QEMUMachineInitArgs *args)
+static void ss10_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[1], args);
+    sun4m_hw_init(&sun4m_hwdefs[1], machine);
 }
 
 /* SPARCserver 600MP hardware initialisation */
-static void ss600mp_init(QEMUMachineInitArgs *args)
+static void ss600mp_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[2], args);
+    sun4m_hw_init(&sun4m_hwdefs[2], machine);
 }
 
 /* SPARCstation 20 hardware initialisation */
-static void ss20_init(QEMUMachineInitArgs *args)
+static void ss20_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[3], args);
+    sun4m_hw_init(&sun4m_hwdefs[3], machine);
 }
 
 /* SPARCstation Voyager hardware initialisation */
-static void vger_init(QEMUMachineInitArgs *args)
+static void vger_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[4], args);
+    sun4m_hw_init(&sun4m_hwdefs[4], machine);
 }
 
 /* SPARCstation LX hardware initialisation */
-static void ss_lx_init(QEMUMachineInitArgs *args)
+static void ss_lx_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[5], args);
+    sun4m_hw_init(&sun4m_hwdefs[5], machine);
 }
 
 /* SPARCstation 4 hardware initialisation */
-static void ss4_init(QEMUMachineInitArgs *args)
+static void ss4_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[6], args);
+    sun4m_hw_init(&sun4m_hwdefs[6], machine);
 }
 
 /* SPARCClassic hardware initialisation */
-static void scls_init(QEMUMachineInitArgs *args)
+static void scls_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[7], args);
+    sun4m_hw_init(&sun4m_hwdefs[7], machine);
 }
 
 /* SPARCbook hardware initialisation */
-static void sbook_init(QEMUMachineInitArgs *args)
+static void sbook_init(MachineState *machine)
 {
-    sun4m_hw_init(&sun4m_hwdefs[8], args);
+    sun4m_hw_init(&sun4m_hwdefs[8], machine);
 }
 
 static QEMUMachine ss5_machine = {
diff --git a/hw/sparc64/sun4u.c b/hw/sparc64/sun4u.c
index 6f271d9..33c311b 100644
--- a/hw/sparc64/sun4u.c
+++ b/hw/sparc64/sun4u.c
@@ -811,7 +811,7 @@
 }
 
 static void sun4uv_init(MemoryRegion *address_space_mem,
-                        QEMUMachineInitArgs *args,
+                        MachineState *machine,
                         const struct hwdef *hwdef)
 {
     SPARCCPU *cpu;
@@ -826,10 +826,10 @@
     FWCfgState *fw_cfg;
 
     /* init CPUs */
-    cpu = cpu_devinit(args->cpu_model, hwdef);
+    cpu = cpu_devinit(machine->cpu_model, hwdef);
 
     /* set up devices */
-    ram_init(0, args->ram_size);
+    ram_init(0, machine->ram_size);
 
     prom_init(hwdef->prom_addr, bios_name);
 
@@ -875,15 +875,15 @@
 
     initrd_size = 0;
     initrd_addr = 0;
-    kernel_size = sun4u_load_kernel(args->kernel_filename,
-                                    args->initrd_filename,
+    kernel_size = sun4u_load_kernel(machine->kernel_filename,
+                                    machine->initrd_filename,
                                     ram_size, &initrd_size, &initrd_addr,
                                     &kernel_addr, &kernel_entry);
 
-    sun4u_NVRAM_set_params(nvram, NVRAM_SIZE, "Sun4u", args->ram_size,
-                           args->boot_order,
+    sun4u_NVRAM_set_params(nvram, NVRAM_SIZE, "Sun4u", machine->ram_size,
+                           machine->boot_order,
                            kernel_addr, kernel_size,
-                           args->kernel_cmdline,
+                           machine->kernel_cmdline,
                            initrd_addr, initrd_size,
                            /* XXX: need an option to load a NVRAM image */
                            0,
@@ -897,16 +897,16 @@
     fw_cfg_add_i16(fw_cfg, FW_CFG_MACHINE_ID, hwdef->machine_id);
     fw_cfg_add_i64(fw_cfg, FW_CFG_KERNEL_ADDR, kernel_entry);
     fw_cfg_add_i64(fw_cfg, FW_CFG_KERNEL_SIZE, kernel_size);
-    if (args->kernel_cmdline) {
+    if (machine->kernel_cmdline) {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE,
-                       strlen(args->kernel_cmdline) + 1);
-        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, args->kernel_cmdline);
+                       strlen(machine->kernel_cmdline) + 1);
+        fw_cfg_add_string(fw_cfg, FW_CFG_CMDLINE_DATA, machine->kernel_cmdline);
     } else {
         fw_cfg_add_i32(fw_cfg, FW_CFG_CMDLINE_SIZE, 0);
     }
     fw_cfg_add_i64(fw_cfg, FW_CFG_INITRD_ADDR, initrd_addr);
     fw_cfg_add_i64(fw_cfg, FW_CFG_INITRD_SIZE, initrd_size);
-    fw_cfg_add_i16(fw_cfg, FW_CFG_BOOT_DEVICE, args->boot_order[0]);
+    fw_cfg_add_i16(fw_cfg, FW_CFG_BOOT_DEVICE, machine->boot_order[0]);
 
     fw_cfg_add_i16(fw_cfg, FW_CFG_SPARC64_WIDTH, graphic_width);
     fw_cfg_add_i16(fw_cfg, FW_CFG_SPARC64_HEIGHT, graphic_height);
@@ -946,21 +946,21 @@
 };
 
 /* Sun4u hardware initialisation */
-static void sun4u_init(QEMUMachineInitArgs *args)
+static void sun4u_init(MachineState *machine)
 {
-    sun4uv_init(get_system_memory(), args, &hwdefs[0]);
+    sun4uv_init(get_system_memory(), machine, &hwdefs[0]);
 }
 
 /* Sun4v hardware initialisation */
-static void sun4v_init(QEMUMachineInitArgs *args)
+static void sun4v_init(MachineState *machine)
 {
-    sun4uv_init(get_system_memory(), args, &hwdefs[1]);
+    sun4uv_init(get_system_memory(), machine, &hwdefs[1]);
 }
 
 /* Niagara hardware initialisation */
-static void niagara_init(QEMUMachineInitArgs *args)
+static void niagara_init(MachineState *machine)
 {
-    sun4uv_init(get_system_memory(), args, &hwdefs[2]);
+    sun4uv_init(get_system_memory(), machine, &hwdefs[2]);
 }
 
 static QEMUMachine sun4u_machine = {
diff --git a/hw/ssi/ssi.c b/hw/ssi/ssi.c
index 1c82a93..2aab79b 100644
--- a/hw/ssi/ssi.c
+++ b/hw/ssi/ssi.c
@@ -60,7 +60,7 @@
 
     if (ssc->transfer_raw == ssi_transfer_raw_default &&
             ssc->cs_polarity != SSI_CS_NONE) {
-        qdev_init_gpio_in(dev, ssi_cs_default, 1);
+        qdev_init_gpio_in_named(dev, ssi_cs_default, SSI_GPIO_CS, 1);
     }
 
     return ssc->init(s);
@@ -155,7 +155,7 @@
         return 0;
     }
 
-    cs_line = qdev_get_gpio_in(DEVICE(dev), 0);
+    cs_line = qdev_get_gpio_in_named(DEVICE(dev), SSI_GPIO_CS, 0);
     qdev_set_parent_bus(DEVICE(dev), BUS(arg->bus));
     **arg->cs_linep = cs_line;
     (*arg->cs_linep)++;
diff --git a/hw/unicore32/puv3.c b/hw/unicore32/puv3.c
index 42913b6..08dd4d0 100644
--- a/hw/unicore32/puv3.c
+++ b/hw/unicore32/puv3.c
@@ -101,12 +101,12 @@
     graphic_console_init(NULL, 0, &no_ops, NULL);
 }
 
-static void puv3_init(QEMUMachineInitArgs *args)
+static void puv3_init(MachineState *machine)
 {
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *initrd_filename = args->initrd_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *initrd_filename = machine->initrd_filename;
     CPUUniCore32State *env;
 
     if (initrd_filename) {
diff --git a/hw/usb/core.c b/hw/usb/core.c
index 67ba7d6..cf34755 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -28,6 +28,26 @@
 #include "qemu/iov.h"
 #include "trace.h"
 
+void usb_pick_speed(USBPort *port)
+{
+    static const int speeds[] = {
+        USB_SPEED_SUPER,
+        USB_SPEED_HIGH,
+        USB_SPEED_FULL,
+        USB_SPEED_LOW,
+    };
+    USBDevice *udev = port->dev;
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(speeds); i++) {
+        if ((udev->speedmask & (1 << speeds[i])) &&
+            (port->speedmask & (1 << speeds[i]))) {
+            udev->speed = speeds[i];
+            return;
+        }
+    }
+}
+
 void usb_attach(USBPort *port)
 {
     USBDevice *dev = port->dev;
@@ -35,6 +55,7 @@
     assert(dev != NULL);
     assert(dev->attached);
     assert(dev->state == USB_STATE_NOTATTACHED);
+    usb_pick_speed(port);
     port->ops->attach(port);
     dev->state = USB_STATE_ATTACHED;
     usb_device_handle_attach(dev);
diff --git a/hw/usb/desc.c b/hw/usb/desc.c
index ab48691..b82c397 100644
--- a/hw/usb/desc.c
+++ b/hw/usb/desc.c
@@ -518,18 +518,6 @@
 
 void usb_desc_attach(USBDevice *dev)
 {
-    const USBDesc *desc = usb_device_get_usb_desc(dev);
-
-    assert(desc != NULL);
-    if (desc->super && (dev->port->speedmask & USB_SPEED_MASK_SUPER)) {
-        dev->speed = USB_SPEED_SUPER;
-    } else if (desc->high && (dev->port->speedmask & USB_SPEED_MASK_HIGH)) {
-        dev->speed = USB_SPEED_HIGH;
-    } else if (desc->full && (dev->port->speedmask & USB_SPEED_MASK_FULL)) {
-        dev->speed = USB_SPEED_FULL;
-    } else {
-        return;
-    }
     usb_desc_setdefaults(dev);
 }
 
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index d097d93..67a57f1 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -47,6 +47,8 @@
     USBEndpoint *intr;
     HIDState hid;
     uint32_t usb_version;
+    char *display;
+    uint32_t head;
 } USBHIDState;
 
 enum {
@@ -574,6 +576,9 @@
     usb_desc_init(dev);
     us->intr = usb_ep_get(dev, USB_TOKEN_IN, 1);
     hid_init(&us->hid, kind, usb_hid_changed);
+    if (us->display && us->hid.s) {
+        qemu_input_handler_bind(us->hid.s, us->display, us->head, NULL);
+    }
     return 0;
 }
 
@@ -653,6 +658,8 @@
 
 static Property usb_tablet_properties[] = {
         DEFINE_PROP_UINT32("usb_version", USBHIDState, usb_version, 2),
+        DEFINE_PROP_STRING("display", USBHIDState, display),
+        DEFINE_PROP_UINT32("head", USBHIDState, head, 0),
         DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -696,6 +703,11 @@
     .class_init    = usb_mouse_class_initfn,
 };
 
+static Property usb_keyboard_properties[] = {
+        DEFINE_PROP_STRING("display", USBHIDState, display),
+        DEFINE_PROP_END_OF_LIST(),
+};
+
 static void usb_keyboard_class_initfn(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
@@ -706,6 +718,7 @@
     uc->product_desc   = "QEMU USB Keyboard";
     uc->usb_desc       = &desc_keyboard;
     dc->vmsd = &vmstate_usb_kbd;
+    dc->props = usb_keyboard_properties;
     set_bit(DEVICE_CATEGORY_INPUT, dc->categories);
 }
 
diff --git a/hw/usb/dev-mtp.c b/hw/usb/dev-mtp.c
index 943f930..380b465 100644
--- a/hw/usb/dev-mtp.c
+++ b/hw/usb/dev-mtp.c
@@ -46,6 +46,7 @@
 
     /* response codes */
     RES_OK                         = 0x2001,
+    RES_GENERAL_ERROR              = 0x2002,
     RES_SESSION_NOT_OPEN           = 0x2003,
     RES_INVALID_TRANSACTION_ID     = 0x2004,
     RES_OPERATION_NOT_SUPPORTED    = 0x2005,
@@ -109,7 +110,8 @@
     struct stat  stat;
     MTPObject    *parent;
     MTPObject    **children;
-    int32_t      nchildren;
+    uint32_t     nchildren;
+    bool         have_children;
     QTAILQ_ENTRY(MTPObject) next;
 };
 
@@ -273,7 +275,6 @@
     o->handle = handle;
     o->parent = parent;
     o->name = g_strdup(name);
-    o->nchildren = -1;
     if (parent == NULL) {
         o->path = g_strdup(name);
     } else {
@@ -340,7 +341,11 @@
     struct dirent *entry;
     DIR *dir;
 
-    o->nchildren = 0;
+    if (o->have_children) {
+        return;
+    }
+    o->have_children = true;
+
     dir = opendir(o->path);
     if (!dir) {
         return;
@@ -698,7 +703,10 @@
     if (offset > o->stat.st_size) {
         offset = o->stat.st_size;
     }
-    lseek(d->fd, offset, SEEK_SET);
+    if (lseek(d->fd, offset, SEEK_SET) < 0) {
+        usb_mtp_data_free(d);
+        return NULL;
+    }
 
     d->length = c->argv[2];
     if (d->length > o->stat.st_size - offset) {
@@ -789,9 +797,7 @@
                                  c->trans, 0, 0, 0);
             return;
         }
-        if (o->nchildren == -1) {
-            usb_mtp_object_readdir(s, o);
-        }
+        usb_mtp_object_readdir(s, o);
         if (c->code == CMD_GET_NUM_OBJECTS) {
             trace_usb_mtp_op_get_num_objects(s->dev.addr, o->handle, o->path);
             nres = 1;
@@ -823,7 +829,9 @@
         }
         data_in = usb_mtp_get_object(s, c, o);
         if (NULL == data_in) {
-            fprintf(stderr, "%s: TODO: handle error\n", __func__);
+            usb_mtp_queue_result(s, RES_GENERAL_ERROR,
+                                 c->trans, 0, 0, 0);
+            return;
         }
         break;
     case CMD_GET_PARTIAL_OBJECT:
@@ -840,7 +848,9 @@
         }
         data_in = usb_mtp_get_partial_object(s, c, o);
         if (NULL == data_in) {
-            fprintf(stderr, "%s: TODO: handle error\n", __func__);
+            usb_mtp_queue_result(s, RES_GENERAL_ERROR,
+                                 c->trans, 0, 0, 0);
+            return;
         }
         nres = 1;
         res0 = data_in->length;
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index a3ae9f2..a00a93c 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -27,87 +27,10 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "hw/usb/ehci-regs.h"
 #include "hw/usb/hcd-ehci.h"
 #include "trace.h"
 
-/* Capability Registers Base Address - section 2.2 */
-#define CAPLENGTH        0x0000  /* 1-byte, 0x0001 reserved */
-#define HCIVERSION       0x0002  /* 2-bytes, i/f version # */
-#define HCSPARAMS        0x0004  /* 4-bytes, structural params */
-#define HCCPARAMS        0x0008  /* 4-bytes, capability params */
-#define EECP             HCCPARAMS + 1
-#define HCSPPORTROUTE1   0x000c
-#define HCSPPORTROUTE2   0x0010
-
-#define USBCMD           0x0000
-#define USBCMD_RUNSTOP   (1 << 0)      // run / Stop
-#define USBCMD_HCRESET   (1 << 1)      // HC Reset
-#define USBCMD_FLS       (3 << 2)      // Frame List Size
-#define USBCMD_FLS_SH    2             // Frame List Size Shift
-#define USBCMD_PSE       (1 << 4)      // Periodic Schedule Enable
-#define USBCMD_ASE       (1 << 5)      // Asynch Schedule Enable
-#define USBCMD_IAAD      (1 << 6)      // Int Asynch Advance Doorbell
-#define USBCMD_LHCR      (1 << 7)      // Light Host Controller Reset
-#define USBCMD_ASPMC     (3 << 8)      // Async Sched Park Mode Count
-#define USBCMD_ASPME     (1 << 11)     // Async Sched Park Mode Enable
-#define USBCMD_ITC       (0x7f << 16)  // Int Threshold Control
-#define USBCMD_ITC_SH    16            // Int Threshold Control Shift
-
-#define USBSTS           0x0004
-#define USBSTS_RO_MASK   0x0000003f
-#define USBSTS_INT       (1 << 0)      // USB Interrupt
-#define USBSTS_ERRINT    (1 << 1)      // Error Interrupt
-#define USBSTS_PCD       (1 << 2)      // Port Change Detect
-#define USBSTS_FLR       (1 << 3)      // Frame List Rollover
-#define USBSTS_HSE       (1 << 4)      // Host System Error
-#define USBSTS_IAA       (1 << 5)      // Interrupt on Async Advance
-#define USBSTS_HALT      (1 << 12)     // HC Halted
-#define USBSTS_REC       (1 << 13)     // Reclamation
-#define USBSTS_PSS       (1 << 14)     // Periodic Schedule Status
-#define USBSTS_ASS       (1 << 15)     // Asynchronous Schedule Status
-
-/*
- *  Interrupt enable bits correspond to the interrupt active bits in USBSTS
- *  so no need to redefine here.
- */
-#define USBINTR              0x0008
-#define USBINTR_MASK         0x0000003f
-
-#define FRINDEX              0x000c
-#define CTRLDSSEGMENT        0x0010
-#define PERIODICLISTBASE     0x0014
-#define ASYNCLISTADDR        0x0018
-#define ASYNCLISTADDR_MASK   0xffffffe0
-
-#define CONFIGFLAG           0x0040
-
-/*
- * Bits that are reserved or are read-only are masked out of values
- * written to us by software
- */
-#define PORTSC_RO_MASK       0x007001c0
-#define PORTSC_RWC_MASK      0x0000002a
-#define PORTSC_WKOC_E        (1 << 22)    // Wake on Over Current Enable
-#define PORTSC_WKDS_E        (1 << 21)    // Wake on Disconnect Enable
-#define PORTSC_WKCN_E        (1 << 20)    // Wake on Connect Enable
-#define PORTSC_PTC           (15 << 16)   // Port Test Control
-#define PORTSC_PTC_SH        16           // Port Test Control shift
-#define PORTSC_PIC           (3 << 14)    // Port Indicator Control
-#define PORTSC_PIC_SH        14           // Port Indicator Control Shift
-#define PORTSC_POWNER        (1 << 13)    // Port Owner
-#define PORTSC_PPOWER        (1 << 12)    // Port Power
-#define PORTSC_LINESTAT      (3 << 10)    // Port Line Status
-#define PORTSC_LINESTAT_SH   10           // Port Line Status Shift
-#define PORTSC_PRESET        (1 << 8)     // Port Reset
-#define PORTSC_SUSPEND       (1 << 7)     // Port Suspend
-#define PORTSC_FPRES         (1 << 6)     // Force Port Resume
-#define PORTSC_OCC           (1 << 5)     // Over Current Change
-#define PORTSC_OCA           (1 << 4)     // Over Current Active
-#define PORTSC_PEDC          (1 << 3)     // Port Enable/Disable Change
-#define PORTSC_PED           (1 << 2)     // Port Enable/Disable
-#define PORTSC_CSC           (1 << 1)     // Connect Status Change
-#define PORTSC_CONNECT       (1 << 0)     // Current Connect Status
-
 #define FRAME_TIMER_FREQ 1000
 #define FRAME_TIMER_NS   (1000000000 / FRAME_TIMER_FREQ)
 #define UFRAME_TIMER_NS  (FRAME_TIMER_NS / 8)
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index 9b1166b..c3bf72c 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -27,6 +27,7 @@
  */
 #include "hw/hw.h"
 #include "hw/usb.h"
+#include "hw/usb/uhci-regs.h"
 #include "hw/pci/pci.h"
 #include "qemu/timer.h"
 #include "qemu/iov.h"
@@ -37,41 +38,6 @@
 //#define DEBUG
 //#define DEBUG_DUMP_DATA
 
-#define UHCI_CMD_FGR      (1 << 4)
-#define UHCI_CMD_EGSM     (1 << 3)
-#define UHCI_CMD_GRESET   (1 << 2)
-#define UHCI_CMD_HCRESET  (1 << 1)
-#define UHCI_CMD_RS       (1 << 0)
-
-#define UHCI_STS_HCHALTED (1 << 5)
-#define UHCI_STS_HCPERR   (1 << 4)
-#define UHCI_STS_HSERR    (1 << 3)
-#define UHCI_STS_RD       (1 << 2)
-#define UHCI_STS_USBERR   (1 << 1)
-#define UHCI_STS_USBINT   (1 << 0)
-
-#define TD_CTRL_SPD     (1 << 29)
-#define TD_CTRL_ERROR_SHIFT  27
-#define TD_CTRL_IOS     (1 << 25)
-#define TD_CTRL_IOC     (1 << 24)
-#define TD_CTRL_ACTIVE  (1 << 23)
-#define TD_CTRL_STALL   (1 << 22)
-#define TD_CTRL_BABBLE  (1 << 20)
-#define TD_CTRL_NAK     (1 << 19)
-#define TD_CTRL_TIMEOUT (1 << 18)
-
-#define UHCI_PORT_SUSPEND (1 << 12)
-#define UHCI_PORT_RESET (1 << 9)
-#define UHCI_PORT_LSDA  (1 << 8)
-#define UHCI_PORT_RD    (1 << 6)
-#define UHCI_PORT_ENC   (1 << 3)
-#define UHCI_PORT_EN    (1 << 2)
-#define UHCI_PORT_CSC   (1 << 1)
-#define UHCI_PORT_CCS   (1 << 0)
-
-#define UHCI_PORT_READ_ONLY    (0x1bb)
-#define UHCI_PORT_WRITE_CLEAR  (UHCI_PORT_CSC | UHCI_PORT_ENC)
-
 #define FRAME_TIMER_FREQ 1000
 
 #define FRAME_MAX_LOOPS  256
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index ef3177a..7f2af89 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -498,6 +498,7 @@
 enum xhci_flags {
     XHCI_FLAG_USE_MSI = 1,
     XHCI_FLAG_USE_MSI_X,
+    XHCI_FLAG_SS_FIRST,
 };
 
 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
@@ -621,6 +622,11 @@
                        ARRAY_SIZE(ep_state_names));
 }
 
+static bool xhci_get_flag(XHCIState *xhci, enum xhci_flags bit)
+{
+    return xhci->flags & (1 << bit);
+}
+
 static uint64_t xhci_mfindex_get(XHCIState *xhci)
 {
     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
@@ -709,10 +715,18 @@
     case USB_SPEED_LOW:
     case USB_SPEED_FULL:
     case USB_SPEED_HIGH:
-        index = uport->index;
+        if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+            index = uport->index + xhci->numports_3;
+        } else {
+            index = uport->index;
+        }
         break;
     case USB_SPEED_SUPER:
-        index = uport->index + xhci->numports_2;
+        if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+            index = uport->index;
+        } else {
+            index = uport->index + xhci->numports_2;
+        }
         break;
     default:
         return NULL;
@@ -2851,7 +2865,7 @@
 
 static void xhci_port_reset(XHCIPort *port, bool warm_reset)
 {
-    trace_usb_xhci_port_reset(port->portnr);
+    trace_usb_xhci_port_reset(port->portnr, warm_reset);
 
     if (!xhci_port_have_device(port)) {
         return;
@@ -2967,7 +2981,11 @@
         ret = 0x20425355; /* "USB " */
         break;
     case 0x28: /* Supported Protocol:08 */
-        ret = 0x00000001 | (xhci->numports_2<<8);
+        if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+            ret = (xhci->numports_2<<8) | (xhci->numports_3+1);
+        } else {
+            ret = (xhci->numports_2<<8) | 1;
+        }
         break;
     case 0x2c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
@@ -2979,7 +2997,11 @@
         ret = 0x20425355; /* "USB " */
         break;
     case 0x38: /* Supported Protocol:08 */
-        ret = 0x00000000 | (xhci->numports_2+1) | (xhci->numports_3<<8);
+        if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+            ret = (xhci->numports_3<<8) | 1;
+        } else {
+            ret = (xhci->numports_3<<8) | (xhci->numports_2+1);
+        }
         break;
     case 0x3c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
@@ -3435,7 +3457,7 @@
     USBBus *bus = usb_bus_from_device(child);
     XHCIState *xhci = container_of(bus, XHCIState, bus);
 
-    xhci_detach_slot(xhci, uport);
+    xhci_detach_slot(xhci, child->port);
 }
 
 static USBPortOps xhci_uport_ops = {
@@ -3512,8 +3534,13 @@
     for (i = 0; i < usbports; i++) {
         speedmask = 0;
         if (i < xhci->numports_2) {
-            port = &xhci->ports[i];
-            port->portnr = i + 1;
+            if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+                port = &xhci->ports[i + xhci->numports_3];
+                port->portnr = i + 1 + xhci->numports_3;
+            } else {
+                port = &xhci->ports[i];
+                port->portnr = i + 1;
+            }
             port->uport = &xhci->uports[i];
             port->speedmask =
                 USB_SPEED_MASK_LOW  |
@@ -3523,8 +3550,13 @@
             speedmask |= port->speedmask;
         }
         if (i < xhci->numports_3) {
-            port = &xhci->ports[i + xhci->numports_2];
-            port->portnr = i + 1 + xhci->numports_2;
+            if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
+                port = &xhci->ports[i];
+                port->portnr = i + 1;
+            } else {
+                port = &xhci->ports[i + xhci->numports_2];
+                port->portnr = i + 1 + xhci->numports_2;
+            }
             port->uport = &xhci->uports[i];
             port->speedmask = USB_SPEED_MASK_SUPER;
             snprintf(port->name, sizeof(port->name), "usb3 port #%d", i+1);
@@ -3594,13 +3626,15 @@
                      PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &xhci->mem);
 
-    ret = pcie_endpoint_cap_init(dev, 0xa0);
-    assert(ret >= 0);
+    if (pci_bus_is_express(dev->bus)) {
+        ret = pcie_endpoint_cap_init(dev, 0xa0);
+        assert(ret >= 0);
+    }
 
-    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI)) {
+    if (xhci_get_flag(xhci, XHCI_FLAG_USE_MSI)) {
         msi_init(dev, 0x70, xhci->numintrs, true, false);
     }
-    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI_X)) {
+    if (xhci_get_flag(xhci, XHCI_FLAG_USE_MSI_X)) {
         msix_init(dev, xhci->numintrs,
                   &xhci->mem, 0, OFF_MSIX_TABLE,
                   &xhci->mem, 0, OFF_MSIX_PBA,
@@ -3781,6 +3815,8 @@
 static Property xhci_properties[] = {
     DEFINE_PROP_BIT("msi",      XHCIState, flags, XHCI_FLAG_USE_MSI, true),
     DEFINE_PROP_BIT("msix",     XHCIState, flags, XHCI_FLAG_USE_MSI_X, true),
+    DEFINE_PROP_BIT("superspeed-ports-first",
+                    XHCIState, flags, XHCI_FLAG_SS_FIRST, true),
     DEFINE_PROP_UINT32("intrs", XHCIState, numintrs, MAXINTRS),
     DEFINE_PROP_UINT32("slots", XHCIState, numslots, MAXSLOTS),
     DEFINE_PROP_UINT32("p2",    XHCIState, numports_2, 4),
diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c
index 57bed09..afbf156 100644
--- a/hw/usb/host-libusb.c
+++ b/hw/usb/host-libusb.c
@@ -111,6 +111,7 @@
     unsigned char                    *buffer;
     unsigned char                    *cbuf;
     unsigned int                     clen;
+    bool                             usb3ep0quirk;
     QTAILQ_ENTRY(USBHostRequest)     next;
 };
 
@@ -146,6 +147,10 @@
 #define BULK_TIMEOUT         0        /* unlimited */
 #define INTR_TIMEOUT         0        /* unlimited */
 
+#if LIBUSBX_API_VERSION >= 0x01000103
+# define HAVE_STREAMS 1
+#endif
+
 static const char *speed_name[] = {
     [LIBUSB_SPEED_UNKNOWN] = "?",
     [LIBUSB_SPEED_LOW]     = "1.5",
@@ -346,6 +351,13 @@
     r->p->actual_length = xfer->actual_length;
     if (r->in && xfer->actual_length) {
         memcpy(r->cbuf, r->buffer + 8, xfer->actual_length);
+
+        /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
+         * to work redirected to a not superspeed capable hcd */
+        if (r->usb3ep0quirk && xfer->actual_length >= 18 &&
+            r->cbuf[7] == 9) {
+            r->cbuf[7] = 64;
+        }
     }
     trace_usb_host_req_complete(s->bus_num, s->addr, r->p,
                                 r->p->status, r->p->actual_length);
@@ -672,11 +684,17 @@
 
 /* ------------------------------------------------------------------------ */
 
-static bool usb_host_full_speed_compat(USBHostDevice *s)
+static void usb_host_speed_compat(USBHostDevice *s)
 {
+    USBDevice *udev = USB_DEVICE(s);
     struct libusb_config_descriptor *conf;
     const struct libusb_interface_descriptor *intf;
     const struct libusb_endpoint_descriptor *endp;
+#ifdef HAVE_STREAMS
+    struct libusb_ss_endpoint_companion_descriptor *endp_ss_comp;
+#endif
+    bool compat_high = true;
+    bool compat_full = true;
     uint8_t type;
     int rc, c, i, a, e;
 
@@ -693,10 +711,27 @@
                     type = endp->bmAttributes & 0x3;
                     switch (type) {
                     case 0x01: /* ISO */
-                        return false;
+                        compat_full = false;
+                        compat_high = false;
+                        break;
+                    case 0x02: /* BULK */
+#ifdef HAVE_STREAMS
+                        rc = libusb_get_ss_endpoint_companion_descriptor
+                            (ctx, endp, &endp_ss_comp);
+                        if (rc == LIBUSB_SUCCESS) {
+                            libusb_free_ss_endpoint_companion_descriptor
+                                (endp_ss_comp);
+                            compat_full = false;
+                            compat_high = false;
+                        }
+#endif
+                        break;
                     case 0x03: /* INTERRUPT */
                         if (endp->wMaxPacketSize > 64) {
-                            return false;
+                            compat_full = false;
+                        }
+                        if (endp->wMaxPacketSize > 1024) {
+                            compat_high = false;
                         }
                         break;
                     }
@@ -705,7 +740,17 @@
         }
         libusb_free_config_descriptor(conf);
     }
-    return true;
+
+    udev->speedmask = (1 << udev->speed);
+    if (udev->speed == USB_SPEED_SUPER && compat_high) {
+        udev->speedmask |= USB_SPEED_HIGH;
+    }
+    if (udev->speed == USB_SPEED_SUPER && compat_full) {
+        udev->speedmask |= USB_SPEED_FULL;
+    }
+    if (udev->speed == USB_SPEED_HIGH && compat_full) {
+        udev->speedmask |= USB_SPEED_FULL;
+    }
 }
 
 static void usb_host_ep_update(USBHostDevice *s)
@@ -720,6 +765,9 @@
     struct libusb_config_descriptor *conf;
     const struct libusb_interface_descriptor *intf;
     const struct libusb_endpoint_descriptor *endp;
+#ifdef HAVE_STREAMS
+    struct libusb_ss_endpoint_companion_descriptor *endp_ss_comp;
+#endif
     uint8_t devep, type;
     int pid, ep;
     int rc, i, e;
@@ -765,6 +813,15 @@
             usb_ep_set_type(udev, pid, ep, type);
             usb_ep_set_ifnum(udev, pid, ep, i);
             usb_ep_set_halted(udev, pid, ep, 0);
+#ifdef HAVE_STREAMS
+            if (type == LIBUSB_TRANSFER_TYPE_BULK &&
+                    libusb_get_ss_endpoint_companion_descriptor(ctx, endp,
+                        &endp_ss_comp) == LIBUSB_SUCCESS) {
+                usb_ep_set_max_streams(udev, pid, ep,
+                                       endp_ss_comp->bmAttributes);
+                libusb_free_ss_endpoint_companion_descriptor(endp_ss_comp);
+            }
+#endif
         }
     }
 
@@ -801,10 +858,7 @@
     usb_host_ep_update(s);
 
     udev->speed     = speed_map[libusb_get_device_speed(dev)];
-    udev->speedmask = (1 << udev->speed);
-    if (udev->speed == USB_SPEED_HIGH && usb_host_full_speed_compat(s)) {
-        udev->speedmask |= USB_SPEED_MASK_FULL;
-    }
+    usb_host_speed_compat(s);
 
     if (s->ddesc.iProduct) {
         libusb_get_string_descriptor_ascii(s->dh, s->ddesc.iProduct,
@@ -1150,6 +1204,14 @@
         memcpy(r->buffer + 8, r->cbuf, r->clen);
     }
 
+    /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
+     * to work redirected to a not superspeed capable hcd */
+    if (udev->speed == USB_SPEED_SUPER &&
+        !((udev->port->speedmask & USB_SPEED_MASK_SUPER)) &&
+        request == 0x8006 && value == 0x100 && index == 0) {
+        r->usb3ep0quirk = true;
+    }
+
     libusb_fill_control_transfer(r->xfer, s->dh, r->buffer,
                                  usb_host_req_complete_ctrl, r,
                                  CONTROL_TIMEOUT);
@@ -1202,10 +1264,23 @@
             usb_packet_copy(p, r->buffer, size);
         }
         ep = p->ep->nr | (r->in ? USB_DIR_IN : 0);
-        libusb_fill_bulk_transfer(r->xfer, s->dh, ep,
-                                  r->buffer, size,
-                                  usb_host_req_complete_data, r,
-                                  BULK_TIMEOUT);
+        if (p->stream) {
+#ifdef HAVE_STREAMS
+            libusb_fill_bulk_stream_transfer(r->xfer, s->dh, ep, p->stream,
+                                             r->buffer, size,
+                                             usb_host_req_complete_data, r,
+                                             BULK_TIMEOUT);
+#else
+            usb_host_req_free(r);
+            p->status = USB_RET_STALL;
+            return;
+#endif
+        } else {
+            libusb_fill_bulk_transfer(r->xfer, s->dh, ep,
+                                      r->buffer, size,
+                                      usb_host_req_complete_data, r,
+                                      BULK_TIMEOUT);
+        }
         break;
     case USB_ENDPOINT_XFER_INT:
         r = usb_host_req_alloc(s, p, p->pid == USB_TOKEN_IN, p->iov.size);
@@ -1268,6 +1343,54 @@
     }
 }
 
+static int usb_host_alloc_streams(USBDevice *udev, USBEndpoint **eps,
+                                  int nr_eps, int streams)
+{
+#ifdef HAVE_STREAMS
+    USBHostDevice *s = USB_HOST_DEVICE(udev);
+    unsigned char endpoints[30];
+    int i, rc;
+
+    for (i = 0; i < nr_eps; i++) {
+        endpoints[i] = eps[i]->nr;
+        if (eps[i]->pid == USB_TOKEN_IN) {
+            endpoints[i] |= 0x80;
+        }
+    }
+    rc = libusb_alloc_streams(s->dh, streams, endpoints, nr_eps);
+    if (rc < 0) {
+        usb_host_libusb_error("libusb_alloc_streams", rc);
+    } else if (rc != streams) {
+        fprintf(stderr,
+            "libusb_alloc_streams: got less streams then requested %d < %d\n",
+            rc, streams);
+    }
+
+    return (rc == streams) ? 0 : -1;
+#else
+    fprintf(stderr, "libusb_alloc_streams: error not implemented\n");
+    return -1;
+#endif
+}
+
+static void usb_host_free_streams(USBDevice *udev, USBEndpoint **eps,
+                                  int nr_eps)
+{
+#ifdef HAVE_STREAMS
+    USBHostDevice *s = USB_HOST_DEVICE(udev);
+    unsigned char endpoints[30];
+    int i;
+
+    for (i = 0; i < nr_eps; i++) {
+        endpoints[i] = eps[i]->nr;
+        if (eps[i]->pid == USB_TOKEN_IN) {
+            endpoints[i] |= 0x80;
+        }
+    }
+    libusb_free_streams(s->dh, endpoints, nr_eps);
+#endif
+}
+
 /*
  * This is *NOT* about restoring state.  We have absolutely no idea
  * what state the host device is in at the moment and whenever it is
@@ -1349,6 +1472,8 @@
     uc->handle_reset   = usb_host_handle_reset;
     uc->handle_destroy = usb_host_handle_destroy;
     uc->flush_ep_queue = usb_host_flush_ep_queue;
+    uc->alloc_streams  = usb_host_alloc_streams;
+    uc->free_streams   = usb_host_free_streams;
     dc->vmsd = &vmstate_usb_host;
     dc->props = usb_host_dev_properties;
     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 287a505..4c6187b 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -50,6 +50,10 @@
                        ((i) & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT, \
                        (i) & 0x0f))
 
+#ifndef USBREDIR_VERSION /* This is not defined in older usbredir versions */
+#define USBREDIR_VERSION 0
+#endif
+
 typedef struct USBRedirDevice USBRedirDevice;
 
 /* Struct to hold buffered packets */
@@ -68,6 +72,7 @@
     uint8_t interval;
     uint8_t interface; /* bInterfaceNumber this ep belongs to */
     uint16_t max_packet_size; /* In bytes, not wMaxPacketSize format !! */
+    uint32_t max_streams;
     uint8_t iso_started;
     uint8_t iso_error; /* For reporting iso errors to the HC */
     uint8_t interrupt_started;
@@ -106,8 +111,9 @@
     int read_buf_size;
     /* Active chardev-watch-tag */
     guint watch;
-    /* For async handling of close */
+    /* For async handling of close / reject */
     QEMUBH *chardev_close_bh;
+    QEMUBH *device_reject_bh;
     /* To delay the usb attach in case of quick chardev close + open */
     QEMUTimer *attach_timer;
     int64_t next_attach_time;
@@ -780,11 +786,12 @@
         dev->endpoint[EP2I(ep)].bulk_receiving_enabled = 0;
     }
 
-    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
+    DPRINTF("bulk-out ep %02X stream %u len %zd id %"PRIu64"\n",
+            ep, p->stream, size, p->id);
 
     bulk_packet.endpoint  = ep;
     bulk_packet.length    = size;
-    bulk_packet.stream_id = 0;
+    bulk_packet.stream_id = p->stream;
     bulk_packet.length_high = size >> 16;
     assert(bulk_packet.length_high == 0 ||
            usbredirparser_peer_has_cap(dev->parser,
@@ -1091,6 +1098,66 @@
     p->status = USB_RET_ASYNC;
 }
 
+static int usbredir_alloc_streams(USBDevice *udev, USBEndpoint **eps,
+                                  int nr_eps, int streams)
+{
+    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+#if USBREDIR_VERSION >= 0x000700
+    struct usb_redir_alloc_bulk_streams_header alloc_streams;
+    int i;
+
+    if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_bulk_streams)) {
+        ERROR("peer does not support streams\n");
+        goto reject;
+    }
+
+    if (streams == 0) {
+        ERROR("request to allocate 0 streams\n");
+        return -1;
+    }
+
+    alloc_streams.no_streams = streams;
+    alloc_streams.endpoints = 0;
+    for (i = 0; i < nr_eps; i++) {
+        alloc_streams.endpoints |= 1 << USBEP2I(eps[i]);
+    }
+    usbredirparser_send_alloc_bulk_streams(dev->parser, 0, &alloc_streams);
+    usbredirparser_do_write(dev->parser);
+
+    return 0;
+#else
+    ERROR("usbredir_alloc_streams not implemented\n");
+    goto reject;
+#endif
+reject:
+    ERROR("streams are not available, disconnecting\n");
+    qemu_bh_schedule(dev->device_reject_bh);
+    return -1;
+}
+
+static void usbredir_free_streams(USBDevice *udev, USBEndpoint **eps,
+                                  int nr_eps)
+{
+#if USBREDIR_VERSION >= 0x000700
+    USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
+    struct usb_redir_free_bulk_streams_header free_streams;
+    int i;
+
+    if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_bulk_streams)) {
+        return;
+    }
+
+    free_streams.endpoints = 0;
+    for (i = 0; i < nr_eps; i++) {
+        free_streams.endpoints |= 1 << USBEP2I(eps[i]);
+    }
+    usbredirparser_send_free_bulk_streams(dev->parser, 0, &free_streams);
+    usbredirparser_do_write(dev->parser);
+#endif
+}
+
 /*
  * Close events can be triggered by usbredirparser_do_write which gets called
  * from within the USBDevice data / control packet callbacks and doing a
@@ -1102,6 +1169,7 @@
 {
     USBRedirDevice *dev = opaque;
 
+    qemu_bh_cancel(dev->device_reject_bh);
     usbredir_device_disconnect(dev);
 
     if (dev->parser) {
@@ -1153,6 +1221,9 @@
     usbredirparser_caps_set_cap(caps, usb_redir_cap_64bits_ids);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
     usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_receiving);
+#if USBREDIR_VERSION >= 0x000700
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_bulk_streams);
+#endif
 
     if (runstate_check(RUN_STATE_INMIGRATE)) {
         flags |= usbredirparser_fl_no_hello;
@@ -1171,6 +1242,17 @@
     }
 }
 
+/*
+ * We may need to reject the device when the hcd calls alloc_streams, doing
+ * an usb_detach from within a hcd call is not a good idea, hence this bh.
+ */
+static void usbredir_device_reject_bh(void *opaque)
+{
+    USBRedirDevice *dev = opaque;
+
+    usbredir_reject_device(dev);
+}
+
 static void usbredir_do_attach(void *opaque)
 {
     USBRedirDevice *dev = opaque;
@@ -1297,6 +1379,7 @@
     }
 
     dev->chardev_close_bh = qemu_bh_new(usbredir_chardev_close_bh, dev);
+    dev->device_reject_bh = qemu_bh_new(usbredir_device_reject_bh, dev);
     dev->attach_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, usbredir_do_attach, dev);
 
     packet_id_queue_init(&dev->cancelled, dev, "cancelled");
@@ -1337,6 +1420,7 @@
     dev->cs = NULL;
     /* Note must be done after qemu_chr_close, as that causes a close event */
     qemu_bh_delete(dev->chardev_close_bh);
+    qemu_bh_delete(dev->device_reject_bh);
 
     timer_del(dev->attach_timer);
     timer_free(dev->attach_timer);
@@ -1628,6 +1712,7 @@
         usb_ep->type = dev->endpoint[i].type;
         usb_ep->ifnum = dev->endpoint[i].interface;
         usb_ep->max_packet_size = dev->endpoint[i].max_packet_size;
+        usb_ep->max_streams = dev->endpoint[i].max_streams;
         usbredir_set_pipeline(dev, usb_ep);
     }
 }
@@ -1646,6 +1731,12 @@
                                      usb_redir_cap_ep_info_max_packet_size)) {
             dev->endpoint[i].max_packet_size = ep_info->max_packet_size[i];
         }
+#if USBREDIR_VERSION >= 0x000700
+        if (usbredirparser_peer_has_cap(dev->parser,
+                                        usb_redir_cap_bulk_streams)) {
+            dev->endpoint[i].max_streams = ep_info->max_streams[i];
+        }
+#endif
         switch (dev->endpoint[i].type) {
         case usb_redir_type_invalid:
             break;
@@ -1779,6 +1870,20 @@
 static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status)
 {
+#if USBREDIR_VERSION >= 0x000700
+    USBRedirDevice *dev = priv;
+
+    if (bulk_streams_status->status == usb_redir_success) {
+        DPRINTF("bulk streams status %d eps %08x\n",
+                bulk_streams_status->status, bulk_streams_status->endpoints);
+    } else {
+        ERROR("bulk streams %s failed status %d eps %08x\n",
+              (bulk_streams_status->no_streams == 0) ? "free" : "alloc",
+              bulk_streams_status->status, bulk_streams_status->endpoints);
+        ERROR("usb-redir-host does not provide streams, disconnecting\n");
+        usbredir_reject_device(dev);
+    }
+#endif
 }
 
 static void usbredir_bulk_receiving_status(void *priv, uint64_t id,
@@ -1850,8 +1955,8 @@
     int len = (bulk_packet->length_high << 16) | bulk_packet->length;
     USBPacket *p;
 
-    DPRINTF("bulk-in status %d ep %02X len %d id %"PRIu64"\n",
-            bulk_packet->status, ep, len, id);
+    DPRINTF("bulk-in status %d ep %02X stream %u len %d id %"PRIu64"\n",
+            bulk_packet->status, ep, bulk_packet->stream_id, len, id);
 
     p = usbredir_find_packet_by_id(dev, ep, id);
     if (p) {
@@ -2165,6 +2270,23 @@
     return endp->bulk_receiving_started;
 }
 
+static const VMStateDescription usbredir_stream_vmstate = {
+    .name = "usb-redir-ep/stream-state",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(max_streams, struct endp_data),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static bool usbredir_stream_needed(void *priv)
+{
+    struct endp_data *endp = priv;
+
+    return endp->max_streams;
+}
+
 static const VMStateDescription usbredir_ep_vmstate = {
     .name = "usb-redir-ep",
     .version_id = 1,
@@ -2197,6 +2319,9 @@
             .vmsd = &usbredir_bulk_receiving_vmstate,
             .needed = usbredir_bulk_receiving_needed,
         }, {
+            .vmsd = &usbredir_stream_vmstate,
+            .needed = usbredir_stream_needed,
+        }, {
             /* empty */
         }
     }
@@ -2361,6 +2486,8 @@
     uc->handle_control = usbredir_handle_control;
     uc->flush_ep_queue = usbredir_flush_ep_queue;
     uc->ep_stopped     = usbredir_ep_stopped;
+    uc->alloc_streams  = usbredir_alloc_streams;
+    uc->free_streams   = usbredir_free_streams;
     dc->vmsd           = &usbredir_vmstate;
     dc->props          = usbredir_properties;
     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
diff --git a/hw/xenpv/xen_machine_pv.c b/hw/xenpv/xen_machine_pv.c
index 9adb57f..3150064 100644
--- a/hw/xenpv/xen_machine_pv.c
+++ b/hw/xenpv/xen_machine_pv.c
@@ -28,11 +28,11 @@
 #include "xen_domainbuild.h"
 #include "sysemu/blockdev.h"
 
-static void xen_init_pv(QEMUMachineInitArgs *args)
+static void xen_init_pv(MachineState *machine)
 {
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
-    const char *initrd_filename = args->initrd_filename;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
+    const char *initrd_filename = machine->initrd_filename;
     DriveInfo *dinfo;
     int i;
 
diff --git a/hw/xtensa/xtensa_lx60.c b/hw/xtensa/xtensa_lx60.c
index 49c58d1..507dd88 100644
--- a/hw/xtensa/xtensa_lx60.c
+++ b/hw/xtensa/xtensa_lx60.c
@@ -159,7 +159,7 @@
     cpu_reset(CPU(cpu));
 }
 
-static void lx_init(const LxBoardDesc *board, QEMUMachineInitArgs *args)
+static void lx_init(const LxBoardDesc *board, MachineState *machine)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     int be = 1;
@@ -172,9 +172,9 @@
     MemoryRegion *ram, *rom, *system_io;
     DriveInfo *dinfo;
     pflash_t *flash = NULL;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
-    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
+    const char *kernel_cmdline = machine->kernel_cmdline;
     int n;
 
     if (!cpu_model) {
@@ -198,7 +198,7 @@
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, NULL, "lx60.dram", args->ram_size);
+    memory_region_init_ram(ram, NULL, "lx60.dram", machine->ram_size);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -275,7 +275,7 @@
     }
 }
 
-static void xtensa_lx60_init(QEMUMachineInitArgs *args)
+static void xtensa_lx60_init(MachineState *machine)
 {
     static const LxBoardDesc lx60_board = {
         .flash_base = 0xf8000000,
@@ -283,10 +283,10 @@
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
     };
-    lx_init(&lx60_board, args);
+    lx_init(&lx60_board, machine);
 }
 
-static void xtensa_lx200_init(QEMUMachineInitArgs *args)
+static void xtensa_lx200_init(MachineState *machine)
 {
     static const LxBoardDesc lx200_board = {
         .flash_base = 0xf8000000,
@@ -294,10 +294,10 @@
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
-    lx_init(&lx200_board, args);
+    lx_init(&lx200_board, machine);
 }
 
-static void xtensa_ml605_init(QEMUMachineInitArgs *args)
+static void xtensa_ml605_init(MachineState *machine)
 {
     static const LxBoardDesc ml605_board = {
         .flash_base = 0xf8000000,
@@ -305,10 +305,10 @@
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
-    lx_init(&ml605_board, args);
+    lx_init(&ml605_board, machine);
 }
 
-static void xtensa_kc705_init(QEMUMachineInitArgs *args)
+static void xtensa_kc705_init(MachineState *machine)
 {
     static const LxBoardDesc kc705_board = {
         .flash_base = 0xf0000000,
@@ -316,7 +316,7 @@
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
-    lx_init(&kc705_board, args);
+    lx_init(&kc705_board, machine);
 }
 
 static QEMUMachine xtensa_lx60_machine = {
diff --git a/hw/xtensa/xtensa_sim.c b/hw/xtensa/xtensa_sim.c
index 1192ce7..89da43c 100644
--- a/hw/xtensa/xtensa_sim.c
+++ b/hw/xtensa/xtensa_sim.c
@@ -46,14 +46,14 @@
     cpu_reset(CPU(cpu));
 }
 
-static void xtensa_sim_init(QEMUMachineInitArgs *args)
+static void xtensa_sim_init(MachineState *machine)
 {
     XtensaCPU *cpu = NULL;
     CPUXtensaState *env = NULL;
     MemoryRegion *ram, *rom;
-    ram_addr_t ram_size = args->ram_size;
-    const char *cpu_model = args->cpu_model;
-    const char *kernel_filename = args->kernel_filename;
+    ram_addr_t ram_size = machine->ram_size;
+    const char *cpu_model = machine->cpu_model;
+    const char *kernel_filename = machine->kernel_filename;
     int n;
 
     if (!cpu_model) {
diff --git a/include/block/block.h b/include/block/block.h
index 1b119aa..faee3aa 100644
--- a/include/block/block.h
+++ b/include/block/block.h
@@ -120,6 +120,8 @@
 /* BDRV_BLOCK_DATA: data is read from bs->file or another file
  * BDRV_BLOCK_ZERO: sectors read as zero
  * BDRV_BLOCK_OFFSET_VALID: sector stored in bs->file as raw data
+ * BDRV_BLOCK_ALLOCATED: the content of the block is determined by this
+ *                       layer (as opposed to the backing file)
  * BDRV_BLOCK_RAW: used internally to indicate that the request
  *                 was answered by the raw driver and that one
  *                 should look in bs->file directly.
@@ -141,10 +143,11 @@
  *  f    t        f       not allocated or unknown offset, read as zero
  *  f    f        f       not allocated or unknown offset, read from backing_hd
  */
-#define BDRV_BLOCK_DATA         1
-#define BDRV_BLOCK_ZERO         2
-#define BDRV_BLOCK_OFFSET_VALID 4
-#define BDRV_BLOCK_RAW          8
+#define BDRV_BLOCK_DATA         0x01
+#define BDRV_BLOCK_ZERO         0x02
+#define BDRV_BLOCK_OFFSET_VALID 0x04
+#define BDRV_BLOCK_RAW          0x08
+#define BDRV_BLOCK_ALLOCATED    0x10
 #define BDRV_BLOCK_OFFSET_MASK  BDRV_SECTOR_MASK
 
 typedef enum {
@@ -159,6 +162,25 @@
     void *opaque;
 } BDRVReopenState;
 
+/*
+ * Block operation types
+ */
+typedef enum BlockOpType {
+    BLOCK_OP_TYPE_BACKUP_SOURCE,
+    BLOCK_OP_TYPE_BACKUP_TARGET,
+    BLOCK_OP_TYPE_CHANGE,
+    BLOCK_OP_TYPE_COMMIT,
+    BLOCK_OP_TYPE_DATAPLANE,
+    BLOCK_OP_TYPE_DRIVE_DEL,
+    BLOCK_OP_TYPE_EJECT,
+    BLOCK_OP_TYPE_EXTERNAL_SNAPSHOT,
+    BLOCK_OP_TYPE_INTERNAL_SNAPSHOT,
+    BLOCK_OP_TYPE_INTERNAL_SNAPSHOT_DELETE,
+    BLOCK_OP_TYPE_MIRROR,
+    BLOCK_OP_TYPE_RESIZE,
+    BLOCK_OP_TYPE_STREAM,
+    BLOCK_OP_TYPE_MAX,
+} BlockOpType;
 
 void bdrv_iostatus_enable(BlockDriverState *bs);
 void bdrv_iostatus_reset(BlockDriverState *bs);
@@ -194,6 +216,7 @@
 int bdrv_open_image(BlockDriverState **pbs, const char *filename,
                     QDict *options, const char *bdref_key, int flags,
                     bool allow_none, Error **errp);
+void bdrv_set_backing_hd(BlockDriverState *bs, BlockDriverState *backing_hd);
 int bdrv_open_backing_file(BlockDriverState *bs, QDict *options, Error **errp);
 void bdrv_append_temp_snapshot(BlockDriverState *bs, int flags, Error **errp);
 int bdrv_open(BlockDriverState **pbs, const char *filename,
@@ -450,8 +473,13 @@
 
 void bdrv_ref(BlockDriverState *bs);
 void bdrv_unref(BlockDriverState *bs);
-void bdrv_set_in_use(BlockDriverState *bs, int in_use);
-int bdrv_in_use(BlockDriverState *bs);
+
+bool bdrv_op_is_blocked(BlockDriverState *bs, BlockOpType op, Error **errp);
+void bdrv_op_block(BlockDriverState *bs, BlockOpType op, Error *reason);
+void bdrv_op_unblock(BlockDriverState *bs, BlockOpType op, Error *reason);
+void bdrv_op_block_all(BlockDriverState *bs, Error *reason);
+void bdrv_op_unblock_all(BlockDriverState *bs, Error *reason);
+bool bdrv_op_blocker_is_empty(BlockDriverState *bs);
 
 #ifdef CONFIG_LINUX_AIO
 int raw_get_aio_fd(BlockDriverState *bs);
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 9ffcb69..f2e753f 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -270,6 +270,8 @@
     size_t opt_mem_alignment;
 } BlockLimits;
 
+typedef struct BdrvOpBlocker BdrvOpBlocker;
+
 /*
  * Note: the function bdrv_append() copies and swaps contents of
  * BlockDriverStates, so if you add new fields to this struct, please
@@ -356,14 +358,20 @@
     QTAILQ_ENTRY(BlockDriverState) device_list;
     QLIST_HEAD(, BdrvDirtyBitmap) dirty_bitmaps;
     int refcnt;
-    int in_use; /* users other than guest access, eg. block migration */
 
     QLIST_HEAD(, BdrvTrackedRequest) tracked_requests;
 
+    /* operation blockers */
+    QLIST_HEAD(, BdrvOpBlocker) op_blockers[BLOCK_OP_TYPE_MAX];
+
     /* long-running background operation */
     BlockJob *job;
 
     QDict *options;
+    BlockdevDetectZeroesOptions detect_zeroes;
+
+    /* The error object in use for blocking operations on backing_hd */
+    Error *backing_blocker;
 };
 
 int get_tmp_filename(char *filename, int size);
diff --git a/include/block/blockjob.h b/include/block/blockjob.h
index d76de62..c0a7875 100644
--- a/include/block/blockjob.h
+++ b/include/block/blockjob.h
@@ -106,6 +106,9 @@
     /** The completion function that will be called when the job completes.  */
     BlockDriverCompletionFunc *cb;
 
+    /** Block other operations when block job is running */
+    Error *blocker;
+
     /** The opaque value that is passed to the completion function.  */
     void *opaque;
 };
diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h
index 9cab592..e8363d7 100644
--- a/include/exec/cpu-all.h
+++ b/include/exec/cpu-all.h
@@ -198,127 +198,8 @@
 #define RESERVED_VA 0ul
 #endif
 
-/* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
-#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + GUEST_BASE))
-
-#if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
-#define h2g_valid(x) 1
-#else
-#define h2g_valid(x) ({ \
-    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
-    (__guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS)) && \
-    (!RESERVED_VA || (__guest < RESERVED_VA)); \
-})
 #endif
 
-#define h2g_nocheck(x) ({ \
-    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
-    (abi_ulong)__ret; \
-})
-
-#define h2g(x) ({ \
-    /* Check if given address fits target address space */ \
-    assert(h2g_valid(x)); \
-    h2g_nocheck(x); \
-})
-
-#define saddr(x) g2h(x)
-#define laddr(x) g2h(x)
-
-#else /* !CONFIG_USER_ONLY */
-/* NOTE: we use double casts if pointers and target_ulong have
-   different sizes */
-#define saddr(x) (uint8_t *)(intptr_t)(x)
-#define laddr(x) (uint8_t *)(intptr_t)(x)
-#endif
-
-#define ldub_raw(p) ldub_p(laddr((p)))
-#define ldsb_raw(p) ldsb_p(laddr((p)))
-#define lduw_raw(p) lduw_p(laddr((p)))
-#define ldsw_raw(p) ldsw_p(laddr((p)))
-#define ldl_raw(p) ldl_p(laddr((p)))
-#define ldq_raw(p) ldq_p(laddr((p)))
-#define ldfl_raw(p) ldfl_p(laddr((p)))
-#define ldfq_raw(p) ldfq_p(laddr((p)))
-#define stb_raw(p, v) stb_p(saddr((p)), v)
-#define stw_raw(p, v) stw_p(saddr((p)), v)
-#define stl_raw(p, v) stl_p(saddr((p)), v)
-#define stq_raw(p, v) stq_p(saddr((p)), v)
-#define stfl_raw(p, v) stfl_p(saddr((p)), v)
-#define stfq_raw(p, v) stfq_p(saddr((p)), v)
-
-
-#if defined(CONFIG_USER_ONLY)
-
-/* if user mode, no other memory access functions */
-#define ldub(p) ldub_raw(p)
-#define ldsb(p) ldsb_raw(p)
-#define lduw(p) lduw_raw(p)
-#define ldsw(p) ldsw_raw(p)
-#define ldl(p) ldl_raw(p)
-#define ldq(p) ldq_raw(p)
-#define ldfl(p) ldfl_raw(p)
-#define ldfq(p) ldfq_raw(p)
-#define stb(p, v) stb_raw(p, v)
-#define stw(p, v) stw_raw(p, v)
-#define stl(p, v) stl_raw(p, v)
-#define stq(p, v) stq_raw(p, v)
-#define stfl(p, v) stfl_raw(p, v)
-#define stfq(p, v) stfq_raw(p, v)
-
-#define cpu_ldub_code(env1, p) ldub_raw(p)
-#define cpu_ldsb_code(env1, p) ldsb_raw(p)
-#define cpu_lduw_code(env1, p) lduw_raw(p)
-#define cpu_ldsw_code(env1, p) ldsw_raw(p)
-#define cpu_ldl_code(env1, p) ldl_raw(p)
-#define cpu_ldq_code(env1, p) ldq_raw(p)
-
-#define cpu_ldub_data(env, addr) ldub_raw(addr)
-#define cpu_lduw_data(env, addr) lduw_raw(addr)
-#define cpu_ldsw_data(env, addr) ldsw_raw(addr)
-#define cpu_ldl_data(env, addr) ldl_raw(addr)
-#define cpu_ldq_data(env, addr) ldq_raw(addr)
-
-#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
-#define cpu_stq_data(env, addr, data) stq_raw(addr, data)
-
-#define cpu_ldub_kernel(env, addr) ldub_raw(addr)
-#define cpu_lduw_kernel(env, addr) lduw_raw(addr)
-#define cpu_ldsw_kernel(env, addr) ldsw_raw(addr)
-#define cpu_ldl_kernel(env, addr) ldl_raw(addr)
-#define cpu_ldq_kernel(env, addr) ldq_raw(addr)
-
-#define cpu_stb_kernel(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_kernel(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_kernel(env, addr, data) stl_raw(addr, data)
-#define cpu_stq_kernel(env, addr, data) stq_raw(addr, data)
-
-#define ldub_kernel(p) ldub_raw(p)
-#define ldsb_kernel(p) ldsb_raw(p)
-#define lduw_kernel(p) lduw_raw(p)
-#define ldsw_kernel(p) ldsw_raw(p)
-#define ldl_kernel(p) ldl_raw(p)
-#define ldq_kernel(p) ldq_raw(p)
-#define ldfl_kernel(p) ldfl_raw(p)
-#define ldfq_kernel(p) ldfq_raw(p)
-#define stb_kernel(p, v) stb_raw(p, v)
-#define stw_kernel(p, v) stw_raw(p, v)
-#define stl_kernel(p, v) stl_raw(p, v)
-#define stq_kernel(p, v) stq_raw(p, v)
-#define stfl_kernel(p, v) stfl_raw(p, v)
-#define stfq_kernel(p, vt) stfq_raw(p, v)
-
-#define cpu_ldub_data(env, addr) ldub_raw(addr)
-#define cpu_lduw_data(env, addr) lduw_raw(addr)
-#define cpu_ldl_data(env, addr) ldl_raw(addr)
-
-#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
-#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
-#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
-#endif /* defined(CONFIG_USER_ONLY) */
-
 /* page related stuff */
 
 #define TARGET_PAGE_SIZE (1 << TARGET_PAGE_BITS)
diff --git a/include/exec/cpu_ldst.h b/include/exec/cpu_ldst.h
new file mode 100644
index 0000000..e5550e7
--- /dev/null
+++ b/include/exec/cpu_ldst.h
@@ -0,0 +1,400 @@
+/*
+ *  Software MMU support
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+/*
+ * Generate inline load/store functions for all MMU modes (typically
+ * at least _user and _kernel) as well as _data versions, for all data
+ * sizes.
+ *
+ * Used by target op helpers.
+ *
+ * MMU mode suffixes are defined in target cpu.h.
+ */
+#ifndef CPU_LDST_H
+#define CPU_LDST_H
+
+#if defined(CONFIG_USER_ONLY)
+/* All direct uses of g2h and h2g need to go away for usermode softmmu.  */
+#define g2h(x) ((void *)((unsigned long)(target_ulong)(x) + GUEST_BASE))
+
+#if HOST_LONG_BITS <= TARGET_VIRT_ADDR_SPACE_BITS
+#define h2g_valid(x) 1
+#else
+#define h2g_valid(x) ({ \
+    unsigned long __guest = (unsigned long)(x) - GUEST_BASE; \
+    (__guest < (1ul << TARGET_VIRT_ADDR_SPACE_BITS)) && \
+    (!RESERVED_VA || (__guest < RESERVED_VA)); \
+})
+#endif
+
+#define h2g_nocheck(x) ({ \
+    unsigned long __ret = (unsigned long)(x) - GUEST_BASE; \
+    (abi_ulong)__ret; \
+})
+
+#define h2g(x) ({ \
+    /* Check if given address fits target address space */ \
+    assert(h2g_valid(x)); \
+    h2g_nocheck(x); \
+})
+
+#define saddr(x) g2h(x)
+#define laddr(x) g2h(x)
+
+#else /* !CONFIG_USER_ONLY */
+/* NOTE: we use double casts if pointers and target_ulong have
+   different sizes */
+#define saddr(x) (uint8_t *)(intptr_t)(x)
+#define laddr(x) (uint8_t *)(intptr_t)(x)
+#endif
+
+#define ldub_raw(p) ldub_p(laddr((p)))
+#define ldsb_raw(p) ldsb_p(laddr((p)))
+#define lduw_raw(p) lduw_p(laddr((p)))
+#define ldsw_raw(p) ldsw_p(laddr((p)))
+#define ldl_raw(p) ldl_p(laddr((p)))
+#define ldq_raw(p) ldq_p(laddr((p)))
+#define ldfl_raw(p) ldfl_p(laddr((p)))
+#define ldfq_raw(p) ldfq_p(laddr((p)))
+#define stb_raw(p, v) stb_p(saddr((p)), v)
+#define stw_raw(p, v) stw_p(saddr((p)), v)
+#define stl_raw(p, v) stl_p(saddr((p)), v)
+#define stq_raw(p, v) stq_p(saddr((p)), v)
+#define stfl_raw(p, v) stfl_p(saddr((p)), v)
+#define stfq_raw(p, v) stfq_p(saddr((p)), v)
+
+
+#if defined(CONFIG_USER_ONLY)
+
+/* if user mode, no other memory access functions */
+#define ldub(p) ldub_raw(p)
+#define ldsb(p) ldsb_raw(p)
+#define lduw(p) lduw_raw(p)
+#define ldsw(p) ldsw_raw(p)
+#define ldl(p) ldl_raw(p)
+#define ldq(p) ldq_raw(p)
+#define ldfl(p) ldfl_raw(p)
+#define ldfq(p) ldfq_raw(p)
+#define stb(p, v) stb_raw(p, v)
+#define stw(p, v) stw_raw(p, v)
+#define stl(p, v) stl_raw(p, v)
+#define stq(p, v) stq_raw(p, v)
+#define stfl(p, v) stfl_raw(p, v)
+#define stfq(p, v) stfq_raw(p, v)
+
+#define cpu_ldub_code(env1, p) ldub_raw(p)
+#define cpu_ldsb_code(env1, p) ldsb_raw(p)
+#define cpu_lduw_code(env1, p) lduw_raw(p)
+#define cpu_ldsw_code(env1, p) ldsw_raw(p)
+#define cpu_ldl_code(env1, p) ldl_raw(p)
+#define cpu_ldq_code(env1, p) ldq_raw(p)
+
+#define cpu_ldub_data(env, addr) ldub_raw(addr)
+#define cpu_lduw_data(env, addr) lduw_raw(addr)
+#define cpu_ldsw_data(env, addr) ldsw_raw(addr)
+#define cpu_ldl_data(env, addr) ldl_raw(addr)
+#define cpu_ldq_data(env, addr) ldq_raw(addr)
+
+#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
+#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
+#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
+#define cpu_stq_data(env, addr, data) stq_raw(addr, data)
+
+#define cpu_ldub_kernel(env, addr) ldub_raw(addr)
+#define cpu_lduw_kernel(env, addr) lduw_raw(addr)
+#define cpu_ldsw_kernel(env, addr) ldsw_raw(addr)
+#define cpu_ldl_kernel(env, addr) ldl_raw(addr)
+#define cpu_ldq_kernel(env, addr) ldq_raw(addr)
+
+#define cpu_stb_kernel(env, addr, data) stb_raw(addr, data)
+#define cpu_stw_kernel(env, addr, data) stw_raw(addr, data)
+#define cpu_stl_kernel(env, addr, data) stl_raw(addr, data)
+#define cpu_stq_kernel(env, addr, data) stq_raw(addr, data)
+
+#define ldub_kernel(p) ldub_raw(p)
+#define ldsb_kernel(p) ldsb_raw(p)
+#define lduw_kernel(p) lduw_raw(p)
+#define ldsw_kernel(p) ldsw_raw(p)
+#define ldl_kernel(p) ldl_raw(p)
+#define ldq_kernel(p) ldq_raw(p)
+#define ldfl_kernel(p) ldfl_raw(p)
+#define ldfq_kernel(p) ldfq_raw(p)
+#define stb_kernel(p, v) stb_raw(p, v)
+#define stw_kernel(p, v) stw_raw(p, v)
+#define stl_kernel(p, v) stl_raw(p, v)
+#define stq_kernel(p, v) stq_raw(p, v)
+#define stfl_kernel(p, v) stfl_raw(p, v)
+#define stfq_kernel(p, vt) stfq_raw(p, v)
+
+#define cpu_ldub_data(env, addr) ldub_raw(addr)
+#define cpu_lduw_data(env, addr) lduw_raw(addr)
+#define cpu_ldl_data(env, addr) ldl_raw(addr)
+
+#define cpu_stb_data(env, addr, data) stb_raw(addr, data)
+#define cpu_stw_data(env, addr, data) stw_raw(addr, data)
+#define cpu_stl_data(env, addr, data) stl_raw(addr, data)
+
+#else
+
+/* XXX: find something cleaner.
+ * Furthermore, this is false for 64 bits targets
+ */
+#define ldul_user       ldl_user
+#define ldul_kernel     ldl_kernel
+#define ldul_hypv       ldl_hypv
+#define ldul_executive  ldl_executive
+#define ldul_supervisor ldl_supervisor
+
+/* The memory helpers for tcg-generated code need tcg_target_long etc.  */
+#include "tcg.h"
+
+uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+
+void helper_stb_mmu(CPUArchState *env, target_ulong addr,
+                    uint8_t val, int mmu_idx);
+void helper_stw_mmu(CPUArchState *env, target_ulong addr,
+                    uint16_t val, int mmu_idx);
+void helper_stl_mmu(CPUArchState *env, target_ulong addr,
+                    uint32_t val, int mmu_idx);
+void helper_stq_mmu(CPUArchState *env, target_ulong addr,
+                    uint64_t val, int mmu_idx);
+
+uint8_t helper_ldb_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint16_t helper_ldw_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint32_t helper_ldl_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
+
+#define CPU_MMU_INDEX 0
+#define MEMSUFFIX MMU_MODE0_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+
+#define CPU_MMU_INDEX 1
+#define MEMSUFFIX MMU_MODE1_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+
+#if (NB_MMU_MODES >= 3)
+
+#define CPU_MMU_INDEX 2
+#define MEMSUFFIX MMU_MODE2_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 3) */
+
+#if (NB_MMU_MODES >= 4)
+
+#define CPU_MMU_INDEX 3
+#define MEMSUFFIX MMU_MODE3_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 4) */
+
+#if (NB_MMU_MODES >= 5)
+
+#define CPU_MMU_INDEX 4
+#define MEMSUFFIX MMU_MODE4_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 5) */
+
+#if (NB_MMU_MODES >= 6)
+
+#define CPU_MMU_INDEX 5
+#define MEMSUFFIX MMU_MODE5_SUFFIX
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif /* (NB_MMU_MODES >= 6) */
+
+#if (NB_MMU_MODES > 6)
+#error "NB_MMU_MODES > 6 is not supported for now"
+#endif /* (NB_MMU_MODES > 6) */
+
+/* these access are slower, they must be as rare as possible */
+#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define MEMSUFFIX _data
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+
+#define ldub(p) ldub_data(p)
+#define ldsb(p) ldsb_data(p)
+#define lduw(p) lduw_data(p)
+#define ldsw(p) ldsw_data(p)
+#define ldl(p) ldl_data(p)
+#define ldq(p) ldq_data(p)
+
+#define stb(p, v) stb_data(p, v)
+#define stw(p, v) stw_data(p, v)
+#define stl(p, v) stl_data(p, v)
+#define stq(p, v) stq_data(p, v)
+
+#define CPU_MMU_INDEX (cpu_mmu_index(env))
+#define MEMSUFFIX _code
+#define SOFTMMU_CODE_ACCESS
+
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#undef SOFTMMU_CODE_ACCESS
+
+/**
+ * tlb_vaddr_to_host:
+ * @env: CPUArchState
+ * @addr: guest virtual address to look up
+ * @access_type: 0 for read, 1 for write, 2 for execute
+ * @mmu_idx: MMU index to use for lookup
+ *
+ * Look up the specified guest virtual index in the TCG softmmu TLB.
+ * If the TLB contains a host virtual address suitable for direct RAM
+ * access, then return it. Otherwise (TLB miss, TLB entry is for an
+ * I/O access, etc) return NULL.
+ *
+ * This is the equivalent of the initial fast-path code used by
+ * TCG backends for guest load and store accesses.
+ */
+static inline void *tlb_vaddr_to_host(CPUArchState *env, target_ulong addr,
+                                      int access_type, int mmu_idx)
+{
+    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
+    CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index];
+    target_ulong tlb_addr;
+    uintptr_t haddr;
+
+    switch (access_type) {
+    case 0:
+        tlb_addr = tlbentry->addr_read;
+        break;
+    case 1:
+        tlb_addr = tlbentry->addr_write;
+        break;
+    case 2:
+        tlb_addr = tlbentry->addr_code;
+        break;
+    default:
+        g_assert_not_reached();
+    }
+
+    if ((addr & TARGET_PAGE_MASK)
+        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
+        /* TLB entry is for a different page */
+        return NULL;
+    }
+
+    if (tlb_addr & ~TARGET_PAGE_MASK) {
+        /* IO access */
+        return NULL;
+    }
+
+    haddr = addr + env->tlb_table[mmu_idx][index].addend;
+    return (void *)haddr;
+}
+
+#endif /* defined(CONFIG_USER_ONLY) */
+
+#endif /* CPU_LDST_H */
diff --git a/include/exec/softmmu_header.h b/include/exec/cpu_ldst_template.h
similarity index 89%
rename from include/exec/softmmu_header.h
rename to include/exec/cpu_ldst_template.h
index d8d9c81..006093a 100644
--- a/include/exec/softmmu_header.h
+++ b/include/exec/cpu_ldst_template.h
@@ -8,7 +8,7 @@
  * 32 and 64 bit cases, also generate floating point functions with
  * the same size.
  *
- * Not used directly but included from softmmu_exec.h and exec-all.h.
+ * Not used directly but included from cpu_ldst.h.
  *
  *  Copyright (c) 2003 Fabrice Bellard
  *
@@ -47,35 +47,18 @@
 #error unsupported data size
 #endif
 
-#if ACCESS_TYPE < (NB_MMU_MODES)
-
-#define CPU_MMU_INDEX ACCESS_TYPE
-#define MMUSUFFIX _mmu
-
-#elif ACCESS_TYPE == (NB_MMU_MODES)
-
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
-#define MMUSUFFIX _mmu
-
-#elif ACCESS_TYPE == (NB_MMU_MODES + 1)
-
-#define CPU_MMU_INDEX (cpu_mmu_index(env))
-#define MMUSUFFIX _cmmu
-
-#else
-#error invalid ACCESS_TYPE
-#endif
-
 #if DATA_SIZE == 8
 #define RES_TYPE uint64_t
 #else
 #define RES_TYPE uint32_t
 #endif
 
-#if ACCESS_TYPE == (NB_MMU_MODES + 1)
+#ifdef SOFTMMU_CODE_ACCESS
 #define ADDR_READ addr_code
+#define MMUSUFFIX _cmmu
 #else
 #define ADDR_READ addr_read
+#define MMUSUFFIX _mmu
 #endif
 
 /* generic load/store macros */
@@ -124,7 +107,7 @@
 }
 #endif
 
-#if ACCESS_TYPE != (NB_MMU_MODES + 1)
+#ifndef SOFTMMU_CODE_ACCESS
 
 /* generic store macro */
 
@@ -148,9 +131,7 @@
     }
 }
 
-#endif /* ACCESS_TYPE != (NB_MMU_MODES + 1) */
 
-#if ACCESS_TYPE != (NB_MMU_MODES + 1)
 
 #if DATA_SIZE == 8
 static inline float64 glue(cpu_ldfq, MEMSUFFIX)(CPUArchState *env,
@@ -200,7 +181,7 @@
 }
 #endif /* DATA_SIZE == 4 */
 
-#endif /* ACCESS_TYPE != (NB_MMU_MODES + 1) */
+#endif /* !SOFTMMU_CODE_ACCESS */
 
 #undef RES_TYPE
 #undef DATA_TYPE
@@ -208,6 +189,5 @@
 #undef SUFFIX
 #undef USUFFIX
 #undef DATA_SIZE
-#undef CPU_MMU_INDEX
 #undef MMUSUFFIX
 #undef ADDR_READ
diff --git a/include/exec/def-helper.h b/include/exec/def-helper.h
deleted file mode 100644
index 255b58b..0000000
--- a/include/exec/def-helper.h
+++ /dev/null
@@ -1,274 +0,0 @@
-/* Helper file for declaring TCG helper functions.
-   Should be included at the start and end of target-foo/helper.h.
-
-   Targets should use DEF_HELPER_N and DEF_HELPER_FLAGS_N to declare helper
-   functions.  Names should be specified without the helper_ prefix, and
-   the return and argument types specified.  3 basic types are understood
-   (i32, i64 and ptr).  Additional aliases are provided for convenience and
-   to match the types used by the C helper implementation.
-
-   The target helper.h should be included in all files that use/define
-   helper functions.  THis will ensure that function prototypes are
-   consistent.  In addition it should be included an extra two times for
-   helper.c, defining:
-    GEN_HELPER 1 to produce op generation functions (gen_helper_*)
-    GEN_HELPER 2 to do runtime registration helper functions.
- */
-
-#ifndef DEF_HELPER_H
-#define DEF_HELPER_H 1
-
-#define HELPER(name) glue(helper_, name)
-
-#define GET_TCGV_i32 GET_TCGV_I32
-#define GET_TCGV_i64 GET_TCGV_I64
-#define GET_TCGV_ptr GET_TCGV_PTR
-
-/* Some types that make sense in C, but not for TCG.  */
-#define dh_alias_i32 i32
-#define dh_alias_s32 i32
-#define dh_alias_int i32
-#define dh_alias_i64 i64
-#define dh_alias_s64 i64
-#define dh_alias_f32 i32
-#define dh_alias_f64 i64
-#if TARGET_LONG_BITS == 32
-#define dh_alias_tl i32
-#else
-#define dh_alias_tl i64
-#endif
-#define dh_alias_ptr ptr
-#define dh_alias_void void
-#define dh_alias_noreturn noreturn
-#define dh_alias_env ptr
-#define dh_alias(t) glue(dh_alias_, t)
-
-#define dh_ctype_i32 uint32_t
-#define dh_ctype_s32 int32_t
-#define dh_ctype_int int
-#define dh_ctype_i64 uint64_t
-#define dh_ctype_s64 int64_t
-#define dh_ctype_f32 float32
-#define dh_ctype_f64 float64
-#define dh_ctype_tl target_ulong
-#define dh_ctype_ptr void *
-#define dh_ctype_void void
-#define dh_ctype_noreturn void QEMU_NORETURN
-#define dh_ctype_env CPUArchState *
-#define dh_ctype(t) dh_ctype_##t
-
-/* We can't use glue() here because it falls foul of C preprocessor
-   recursive expansion rules.  */
-#define dh_retvar_decl0_void void
-#define dh_retvar_decl0_noreturn void
-#define dh_retvar_decl0_i32 TCGv_i32 retval
-#define dh_retvar_decl0_i64 TCGv_i64 retval
-#define dh_retvar_decl0_ptr TCGv_ptr retval
-#define dh_retvar_decl0(t) glue(dh_retvar_decl0_, dh_alias(t))
-
-#define dh_retvar_decl_void
-#define dh_retvar_decl_noreturn
-#define dh_retvar_decl_i32 TCGv_i32 retval,
-#define dh_retvar_decl_i64 TCGv_i64 retval,
-#define dh_retvar_decl_ptr TCGv_ptr retval,
-#define dh_retvar_decl(t) glue(dh_retvar_decl_, dh_alias(t))
-
-#define dh_retvar_void TCG_CALL_DUMMY_ARG
-#define dh_retvar_noreturn TCG_CALL_DUMMY_ARG
-#define dh_retvar_i32 GET_TCGV_i32(retval)
-#define dh_retvar_i64 GET_TCGV_i64(retval)
-#define dh_retvar_ptr GET_TCGV_ptr(retval)
-#define dh_retvar(t) glue(dh_retvar_, dh_alias(t))
-
-#define dh_is_64bit_void 0
-#define dh_is_64bit_noreturn 0
-#define dh_is_64bit_i32 0
-#define dh_is_64bit_i64 1
-#define dh_is_64bit_ptr (sizeof(void *) == 8)
-#define dh_is_64bit(t) glue(dh_is_64bit_, dh_alias(t))
-
-#define dh_is_signed_void 0
-#define dh_is_signed_noreturn 0
-#define dh_is_signed_i32 0
-#define dh_is_signed_s32 1
-#define dh_is_signed_i64 0
-#define dh_is_signed_s64 1
-#define dh_is_signed_f32 0
-#define dh_is_signed_f64 0
-#define dh_is_signed_tl  0
-#define dh_is_signed_int 1
-/* ??? This is highly specific to the host cpu.  There are even special
-   extension instructions that may be required, e.g. ia64's addp4.  But
-   for now we don't support any 64-bit targets with 32-bit pointers.  */
-#define dh_is_signed_ptr 0
-#define dh_is_signed_env dh_is_signed_ptr
-#define dh_is_signed(t) dh_is_signed_##t
-
-#define dh_sizemask(t, n) \
-  sizemask |= dh_is_64bit(t) << (n*2); \
-  sizemask |= dh_is_signed(t) << (n*2+1)
-
-#define dh_arg(t, n) \
-  args[n - 1] = glue(GET_TCGV_, dh_alias(t))(glue(arg, n)); \
-  dh_sizemask(t, n)
-
-#define dh_arg_decl(t, n) glue(TCGv_, dh_alias(t)) glue(arg, n)
-
-
-#define DEF_HELPER_0(name, ret) \
-    DEF_HELPER_FLAGS_0(name, 0, ret)
-#define DEF_HELPER_1(name, ret, t1) \
-    DEF_HELPER_FLAGS_1(name, 0, ret, t1)
-#define DEF_HELPER_2(name, ret, t1, t2) \
-    DEF_HELPER_FLAGS_2(name, 0, ret, t1, t2)
-#define DEF_HELPER_3(name, ret, t1, t2, t3) \
-    DEF_HELPER_FLAGS_3(name, 0, ret, t1, t2, t3)
-#define DEF_HELPER_4(name, ret, t1, t2, t3, t4) \
-    DEF_HELPER_FLAGS_4(name, 0, ret, t1, t2, t3, t4)
-#define DEF_HELPER_5(name, ret, t1, t2, t3, t4, t5) \
-    DEF_HELPER_FLAGS_5(name, 0, ret, t1, t2, t3, t4, t5)
-
-/* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */
-
-#endif /* DEF_HELPER_H */
-
-#ifndef GEN_HELPER
-/* Function prototypes.  */
-
-#define DEF_HELPER_FLAGS_0(name, flags, ret) \
-dh_ctype(ret) HELPER(name) (void);
-
-#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \
-dh_ctype(ret) HELPER(name) (dh_ctype(t1));
-
-#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
-dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2));
-
-#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \
-dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3));
-
-#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \
-dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \
-                                   dh_ctype(t4));
-
-#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \
-dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \
-                            dh_ctype(t4), dh_ctype(t5));
-
-#undef GEN_HELPER
-#define GEN_HELPER -1
-
-#elif GEN_HELPER == 1
-/* Gen functions.  */
-
-#define DEF_HELPER_FLAGS_0(name, flags, ret) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret)) \
-{ \
-  int sizemask; \
-  sizemask = dh_is_64bit(ret); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 0, NULL); \
-}
-
-#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) dh_arg_decl(t1, 1)) \
-{ \
-  TCGArg args[1]; \
-  int sizemask = 0; \
-  dh_sizemask(ret, 0); \
-  dh_arg(t1, 1); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 1, args); \
-}
-
-#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) dh_arg_decl(t1, 1), \
-    dh_arg_decl(t2, 2)) \
-{ \
-  TCGArg args[2]; \
-  int sizemask = 0; \
-  dh_sizemask(ret, 0); \
-  dh_arg(t1, 1); \
-  dh_arg(t2, 2); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 2, args); \
-}
-
-#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) dh_arg_decl(t1, 1), \
-    dh_arg_decl(t2, 2), dh_arg_decl(t3, 3)) \
-{ \
-  TCGArg args[3]; \
-  int sizemask = 0; \
-  dh_sizemask(ret, 0); \
-  dh_arg(t1, 1); \
-  dh_arg(t2, 2); \
-  dh_arg(t3, 3); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 3, args); \
-}
-
-#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) dh_arg_decl(t1, 1), \
-    dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), dh_arg_decl(t4, 4)) \
-{ \
-  TCGArg args[4]; \
-  int sizemask = 0; \
-  dh_sizemask(ret, 0); \
-  dh_arg(t1, 1); \
-  dh_arg(t2, 2); \
-  dh_arg(t3, 3); \
-  dh_arg(t4, 4); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 4, args); \
-}
-
-#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \
-static inline void glue(gen_helper_, name)(dh_retvar_decl(ret) \
-    dh_arg_decl(t1, 1),  dh_arg_decl(t2, 2), dh_arg_decl(t3, 3), \
-    dh_arg_decl(t4, 4), dh_arg_decl(t5, 5)) \
-{ \
-  TCGArg args[5]; \
-  int sizemask = 0; \
-  dh_sizemask(ret, 0); \
-  dh_arg(t1, 1); \
-  dh_arg(t2, 2); \
-  dh_arg(t3, 3); \
-  dh_arg(t4, 4); \
-  dh_arg(t5, 5); \
-  tcg_gen_helperN(HELPER(name), flags, sizemask, dh_retvar(ret), 5, args); \
-}
-
-#undef GEN_HELPER
-#define GEN_HELPER -1
-
-#elif GEN_HELPER == 2
-/* Register helpers.  */
-
-#define DEF_HELPER_FLAGS_0(name, flags, ret)  { HELPER(name), #name },
-
-#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \
-DEF_HELPER_FLAGS_0(name, flags, ret)
-
-#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
-DEF_HELPER_FLAGS_0(name, flags, ret)
-
-#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \
-DEF_HELPER_FLAGS_0(name, flags, ret)
-
-#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \
-DEF_HELPER_FLAGS_0(name, flags, ret)
-
-#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \
-DEF_HELPER_FLAGS_0(name, flags, ret)
-
-#undef GEN_HELPER
-#define GEN_HELPER -1
-
-#elif GEN_HELPER == -1
-/* Undefine macros.  */
-
-#undef DEF_HELPER_FLAGS_0
-#undef DEF_HELPER_FLAGS_1
-#undef DEF_HELPER_FLAGS_2
-#undef DEF_HELPER_FLAGS_3
-#undef DEF_HELPER_FLAGS_4
-#undef DEF_HELPER_FLAGS_5
-#undef GEN_HELPER
-
-#endif
diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h
index 8bc2eb6..3d62d9c 100644
--- a/include/exec/exec-all.h
+++ b/include/exec/exec-all.h
@@ -131,7 +131,7 @@
 #if defined(__arm__) || defined(_ARCH_PPC) \
     || defined(__x86_64__) || defined(__i386__) \
     || defined(__sparc__) || defined(__aarch64__) \
-    || defined(__s390x__) \
+    || defined(__s390x__) || defined(__mips__) \
     || defined(CONFIG_TCG_INTERPRETER)
 #define USE_DIRECT_JUMP
 #endif
@@ -268,7 +268,7 @@
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
-#elif defined(__sparc__)
+#elif defined(__sparc__) || defined(__mips__)
 void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
 #else
 #error tb_set_jmp_target1 is missing
@@ -344,29 +344,6 @@
 void tlb_fill(CPUState *cpu, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr);
 
-uint8_t helper_ldb_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
-#define ACCESS_TYPE (NB_MMU_MODES + 1)
-#define MEMSUFFIX _code
-
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-
 #endif
 
 #if defined(CONFIG_USER_ONLY)
diff --git a/include/exec/helper-gen.h b/include/exec/helper-gen.h
new file mode 100644
index 0000000..a04a034
--- /dev/null
+++ b/include/exec/helper-gen.h
@@ -0,0 +1,70 @@
+/* Helper file for declaring TCG helper functions.
+   This one expands generation functions for tcg opcodes.  */
+
+#ifndef HELPER_GEN_H
+#define HELPER_GEN_H 1
+
+#include <exec/helper-head.h>
+
+#define DEF_HELPER_FLAGS_0(name, flags, ret)                            \
+static inline void glue(gen_helper_, name)(dh_retvar_decl0(ret))        \
+{                                                                       \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 0, NULL);       \
+}
+
+#define DEF_HELPER_FLAGS_1(name, flags, ret, t1)                        \
+static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
+    dh_arg_decl(t1, 1))                                                 \
+{                                                                       \
+  TCGArg args[1] = { dh_arg(t1, 1) };                                   \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 1, args);       \
+}
+
+#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2)                    \
+static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
+    dh_arg_decl(t1, 1), dh_arg_decl(t2, 2))                             \
+{                                                                       \
+  TCGArg args[2] = { dh_arg(t1, 1), dh_arg(t2, 2) };                    \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 2, args);       \
+}
+
+#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3)                \
+static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
+    dh_arg_decl(t1, 1), dh_arg_decl(t2, 2), dh_arg_decl(t3, 3))         \
+{                                                                       \
+  TCGArg args[3] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3) };     \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 3, args);       \
+}
+
+#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4)            \
+static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
+    dh_arg_decl(t1, 1), dh_arg_decl(t2, 2),                             \
+    dh_arg_decl(t3, 3), dh_arg_decl(t4, 4))                             \
+{                                                                       \
+  TCGArg args[4] = { dh_arg(t1, 1), dh_arg(t2, 2),                      \
+                     dh_arg(t3, 3), dh_arg(t4, 4) };                    \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 4, args);       \
+}
+
+#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5)        \
+static inline void glue(gen_helper_, name)(dh_retvar_decl(ret)          \
+    dh_arg_decl(t1, 1),  dh_arg_decl(t2, 2), dh_arg_decl(t3, 3),        \
+    dh_arg_decl(t4, 4), dh_arg_decl(t5, 5))                             \
+{                                                                       \
+  TCGArg args[5] = { dh_arg(t1, 1), dh_arg(t2, 2), dh_arg(t3, 3),       \
+                     dh_arg(t4, 4), dh_arg(t5, 5) };                    \
+  tcg_gen_callN(&tcg_ctx, HELPER(name), dh_retvar(ret), 5, args);       \
+}
+
+#include "helper.h"
+#include "tcg-runtime.h"
+
+#undef DEF_HELPER_FLAGS_0
+#undef DEF_HELPER_FLAGS_1
+#undef DEF_HELPER_FLAGS_2
+#undef DEF_HELPER_FLAGS_3
+#undef DEF_HELPER_FLAGS_4
+#undef DEF_HELPER_FLAGS_5
+#undef GEN_HELPER
+
+#endif /* HELPER_GEN_H */
diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h
new file mode 100644
index 0000000..b009ccb
--- /dev/null
+++ b/include/exec/helper-head.h
@@ -0,0 +1,134 @@
+/* Helper file for declaring TCG helper functions.
+   Used by other helper files.
+
+   Targets should use DEF_HELPER_N and DEF_HELPER_FLAGS_N to declare helper
+   functions.  Names should be specified without the helper_ prefix, and
+   the return and argument types specified.  3 basic types are understood
+   (i32, i64 and ptr).  Additional aliases are provided for convenience and
+   to match the types used by the C helper implementation.
+
+   The target helper.h should be included in all files that use/define
+   helper functions.  THis will ensure that function prototypes are
+   consistent.  In addition it should be included an extra two times for
+   helper.c, defining:
+    GEN_HELPER 1 to produce op generation functions (gen_helper_*)
+    GEN_HELPER 2 to do runtime registration helper functions.
+ */
+
+#ifndef DEF_HELPER_H
+#define DEF_HELPER_H 1
+
+#include "qemu/osdep.h"
+
+#define HELPER(name) glue(helper_, name)
+
+#define GET_TCGV_i32 GET_TCGV_I32
+#define GET_TCGV_i64 GET_TCGV_I64
+#define GET_TCGV_ptr GET_TCGV_PTR
+
+/* Some types that make sense in C, but not for TCG.  */
+#define dh_alias_i32 i32
+#define dh_alias_s32 i32
+#define dh_alias_int i32
+#define dh_alias_i64 i64
+#define dh_alias_s64 i64
+#define dh_alias_f32 i32
+#define dh_alias_f64 i64
+#ifdef TARGET_LONG_BITS
+# if TARGET_LONG_BITS == 32
+#  define dh_alias_tl i32
+# else
+#  define dh_alias_tl i64
+# endif
+#endif
+#define dh_alias_ptr ptr
+#define dh_alias_void void
+#define dh_alias_noreturn noreturn
+#define dh_alias_env ptr
+#define dh_alias(t) glue(dh_alias_, t)
+
+#define dh_ctype_i32 uint32_t
+#define dh_ctype_s32 int32_t
+#define dh_ctype_int int
+#define dh_ctype_i64 uint64_t
+#define dh_ctype_s64 int64_t
+#define dh_ctype_f32 float32
+#define dh_ctype_f64 float64
+#define dh_ctype_tl target_ulong
+#define dh_ctype_ptr void *
+#define dh_ctype_void void
+#define dh_ctype_noreturn void QEMU_NORETURN
+#define dh_ctype_env CPUArchState *
+#define dh_ctype(t) dh_ctype_##t
+
+/* We can't use glue() here because it falls foul of C preprocessor
+   recursive expansion rules.  */
+#define dh_retvar_decl0_void void
+#define dh_retvar_decl0_noreturn void
+#define dh_retvar_decl0_i32 TCGv_i32 retval
+#define dh_retvar_decl0_i64 TCGv_i64 retval
+#define dh_retvar_decl0_ptr TCGv_ptr retval
+#define dh_retvar_decl0(t) glue(dh_retvar_decl0_, dh_alias(t))
+
+#define dh_retvar_decl_void
+#define dh_retvar_decl_noreturn
+#define dh_retvar_decl_i32 TCGv_i32 retval,
+#define dh_retvar_decl_i64 TCGv_i64 retval,
+#define dh_retvar_decl_ptr TCGv_ptr retval,
+#define dh_retvar_decl(t) glue(dh_retvar_decl_, dh_alias(t))
+
+#define dh_retvar_void TCG_CALL_DUMMY_ARG
+#define dh_retvar_noreturn TCG_CALL_DUMMY_ARG
+#define dh_retvar_i32 GET_TCGV_i32(retval)
+#define dh_retvar_i64 GET_TCGV_i64(retval)
+#define dh_retvar_ptr GET_TCGV_ptr(retval)
+#define dh_retvar(t) glue(dh_retvar_, dh_alias(t))
+
+#define dh_is_64bit_void 0
+#define dh_is_64bit_noreturn 0
+#define dh_is_64bit_i32 0
+#define dh_is_64bit_i64 1
+#define dh_is_64bit_ptr (sizeof(void *) == 8)
+#define dh_is_64bit(t) glue(dh_is_64bit_, dh_alias(t))
+
+#define dh_is_signed_void 0
+#define dh_is_signed_noreturn 0
+#define dh_is_signed_i32 0
+#define dh_is_signed_s32 1
+#define dh_is_signed_i64 0
+#define dh_is_signed_s64 1
+#define dh_is_signed_f32 0
+#define dh_is_signed_f64 0
+#define dh_is_signed_tl  0
+#define dh_is_signed_int 1
+/* ??? This is highly specific to the host cpu.  There are even special
+   extension instructions that may be required, e.g. ia64's addp4.  But
+   for now we don't support any 64-bit targets with 32-bit pointers.  */
+#define dh_is_signed_ptr 0
+#define dh_is_signed_env dh_is_signed_ptr
+#define dh_is_signed(t) dh_is_signed_##t
+
+#define dh_sizemask(t, n) \
+  ((dh_is_64bit(t) << (n*2)) | (dh_is_signed(t) << (n*2+1)))
+
+#define dh_arg(t, n) \
+  glue(GET_TCGV_, dh_alias(t))(glue(arg, n))
+
+#define dh_arg_decl(t, n) glue(TCGv_, dh_alias(t)) glue(arg, n)
+
+#define DEF_HELPER_0(name, ret) \
+    DEF_HELPER_FLAGS_0(name, 0, ret)
+#define DEF_HELPER_1(name, ret, t1) \
+    DEF_HELPER_FLAGS_1(name, 0, ret, t1)
+#define DEF_HELPER_2(name, ret, t1, t2) \
+    DEF_HELPER_FLAGS_2(name, 0, ret, t1, t2)
+#define DEF_HELPER_3(name, ret, t1, t2, t3) \
+    DEF_HELPER_FLAGS_3(name, 0, ret, t1, t2, t3)
+#define DEF_HELPER_4(name, ret, t1, t2, t3, t4) \
+    DEF_HELPER_FLAGS_4(name, 0, ret, t1, t2, t3, t4)
+#define DEF_HELPER_5(name, ret, t1, t2, t3, t4, t5) \
+    DEF_HELPER_FLAGS_5(name, 0, ret, t1, t2, t3, t4, t5)
+
+/* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */
+
+#endif /* DEF_HELPER_H */
diff --git a/include/exec/helper-proto.h b/include/exec/helper-proto.h
new file mode 100644
index 0000000..828951c
--- /dev/null
+++ b/include/exec/helper-proto.h
@@ -0,0 +1,39 @@
+/* Helper file for declaring TCG helper functions.
+   This one expands prototypes for the helper functions.  */
+
+#ifndef HELPER_PROTO_H
+#define HELPER_PROTO_H 1
+
+#include <exec/helper-head.h>
+
+#define DEF_HELPER_FLAGS_0(name, flags, ret) \
+dh_ctype(ret) HELPER(name) (void);
+
+#define DEF_HELPER_FLAGS_1(name, flags, ret, t1) \
+dh_ctype(ret) HELPER(name) (dh_ctype(t1));
+
+#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
+dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2));
+
+#define DEF_HELPER_FLAGS_3(name, flags, ret, t1, t2, t3) \
+dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3));
+
+#define DEF_HELPER_FLAGS_4(name, flags, ret, t1, t2, t3, t4) \
+dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \
+                                   dh_ctype(t4));
+
+#define DEF_HELPER_FLAGS_5(name, flags, ret, t1, t2, t3, t4, t5) \
+dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2), dh_ctype(t3), \
+                            dh_ctype(t4), dh_ctype(t5));
+
+#include "helper.h"
+#include "tcg-runtime.h"
+
+#undef DEF_HELPER_FLAGS_0
+#undef DEF_HELPER_FLAGS_1
+#undef DEF_HELPER_FLAGS_2
+#undef DEF_HELPER_FLAGS_3
+#undef DEF_HELPER_FLAGS_4
+#undef DEF_HELPER_FLAGS_5
+
+#endif /* HELPER_PROTO_H */
diff --git a/include/exec/helper-tcg.h b/include/exec/helper-tcg.h
new file mode 100644
index 0000000..d704c81
--- /dev/null
+++ b/include/exec/helper-tcg.h
@@ -0,0 +1,48 @@
+/* Helper file for declaring TCG helper functions.
+   This one defines data structures private to tcg.c.  */
+
+#ifndef HELPER_TCG_H
+#define HELPER_TCG_H 1
+
+#include <exec/helper-head.h>
+
+#define DEF_HELPER_FLAGS_0(NAME, FLAGS, ret) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) },
+
+#define DEF_HELPER_FLAGS_1(NAME, FLAGS, ret, t1) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) },
+
+#define DEF_HELPER_FLAGS_2(NAME, FLAGS, ret, t1, t2) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
+    | dh_sizemask(t2, 2) },
+
+#define DEF_HELPER_FLAGS_3(NAME, FLAGS, ret, t1, t2, t3) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
+    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) },
+
+#define DEF_HELPER_FLAGS_4(NAME, FLAGS, ret, t1, t2, t3, t4) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
+    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) },
+
+#define DEF_HELPER_FLAGS_5(NAME, FLAGS, ret, t1, t2, t3, t4, t5) \
+  { .func = HELPER(NAME), .name = #NAME, .flags = FLAGS, \
+    .sizemask = dh_sizemask(ret, 0) | dh_sizemask(t1, 1) \
+    | dh_sizemask(t2, 2) | dh_sizemask(t3, 3) | dh_sizemask(t4, 4) \
+    | dh_sizemask(t5, 5) },
+
+#include "helper.h"
+#include "tcg-runtime.h"
+
+#undef DEF_HELPER_FLAGS_0
+#undef DEF_HELPER_FLAGS_1
+#undef DEF_HELPER_FLAGS_2
+#undef DEF_HELPER_FLAGS_3
+#undef DEF_HELPER_FLAGS_4
+#undef DEF_HELPER_FLAGS_5
+
+#endif /* HELPER_TCG_H */
diff --git a/include/exec/softmmu_exec.h b/include/exec/softmmu_exec.h
deleted file mode 100644
index 470db20..0000000
--- a/include/exec/softmmu_exec.h
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- *  Software MMU support
- *
- * Generate inline load/store functions for all MMU modes (typically
- * at least _user and _kernel) as well as _data versions, for all data
- * sizes.
- *
- * Used by target op helpers.
- *
- * MMU mode suffixes are defined in target cpu.h.
- */
-
-/* XXX: find something cleaner.
- * Furthermore, this is false for 64 bits targets
- */
-#define ldul_user       ldl_user
-#define ldul_kernel     ldl_kernel
-#define ldul_hypv       ldl_hypv
-#define ldul_executive  ldl_executive
-#define ldul_supervisor ldl_supervisor
-
-/* The memory helpers for tcg-generated code need tcg_target_long etc.  */
-#include "tcg.h"
-
-#define ACCESS_TYPE 0
-#define MEMSUFFIX MMU_MODE0_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-
-#define ACCESS_TYPE 1
-#define MEMSUFFIX MMU_MODE1_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-
-#if (NB_MMU_MODES >= 3)
-
-#define ACCESS_TYPE 2
-#define MEMSUFFIX MMU_MODE2_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-#endif /* (NB_MMU_MODES >= 3) */
-
-#if (NB_MMU_MODES >= 4)
-
-#define ACCESS_TYPE 3
-#define MEMSUFFIX MMU_MODE3_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-#endif /* (NB_MMU_MODES >= 4) */
-
-#if (NB_MMU_MODES >= 5)
-
-#define ACCESS_TYPE 4
-#define MEMSUFFIX MMU_MODE4_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-#endif /* (NB_MMU_MODES >= 5) */
-
-#if (NB_MMU_MODES >= 6)
-
-#define ACCESS_TYPE 5
-#define MEMSUFFIX MMU_MODE5_SUFFIX
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-#endif /* (NB_MMU_MODES >= 6) */
-
-#if (NB_MMU_MODES > 6)
-#error "NB_MMU_MODES > 6 is not supported for now"
-#endif /* (NB_MMU_MODES > 6) */
-
-/* these access are slower, they must be as rare as possible */
-#define ACCESS_TYPE (NB_MMU_MODES)
-#define MEMSUFFIX _data
-#define DATA_SIZE 1
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 2
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 4
-#include "exec/softmmu_header.h"
-
-#define DATA_SIZE 8
-#include "exec/softmmu_header.h"
-#undef ACCESS_TYPE
-#undef MEMSUFFIX
-
-#define ldub(p) ldub_data(p)
-#define ldsb(p) ldsb_data(p)
-#define lduw(p) lduw_data(p)
-#define ldsw(p) ldsw_data(p)
-#define ldl(p) ldl_data(p)
-#define ldq(p) ldq_data(p)
-
-#define stb(p, v) stb_data(p, v)
-#define stw(p, v) stw_data(p, v)
-#define stl(p, v) stl_data(p, v)
-#define stq(p, v) stq_data(p, v)
-
-/**
- * tlb_vaddr_to_host:
- * @env: CPUArchState
- * @addr: guest virtual address to look up
- * @access_type: 0 for read, 1 for write, 2 for execute
- * @mmu_idx: MMU index to use for lookup
- *
- * Look up the specified guest virtual index in the TCG softmmu TLB.
- * If the TLB contains a host virtual address suitable for direct RAM
- * access, then return it. Otherwise (TLB miss, TLB entry is for an
- * I/O access, etc) return NULL.
- *
- * This is the equivalent of the initial fast-path code used by
- * TCG backends for guest load and store accesses.
- */
-static inline void *tlb_vaddr_to_host(CPUArchState *env, target_ulong addr,
-                                      int access_type, int mmu_idx)
-{
-    int index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
-    CPUTLBEntry *tlbentry = &env->tlb_table[mmu_idx][index];
-    target_ulong tlb_addr;
-    uintptr_t haddr;
-
-    switch (access_type) {
-    case 0:
-        tlb_addr = tlbentry->addr_read;
-        break;
-    case 1:
-        tlb_addr = tlbentry->addr_write;
-        break;
-    case 2:
-        tlb_addr = tlbentry->addr_code;
-        break;
-    default:
-        g_assert_not_reached();
-    }
-
-    if ((addr & TARGET_PAGE_MASK)
-        != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
-        /* TLB entry is for a different page */
-        return NULL;
-    }
-
-    if (tlb_addr & ~TARGET_PAGE_MASK) {
-        /* IO access */
-        return NULL;
-    }
-
-    haddr = addr + env->tlb_table[mmu_idx][index].addend;
-    return (void *)haddr;
-}
diff --git a/include/hw/boards.h b/include/hw/boards.h
index 4345bd0..2d2e2be 100644
--- a/include/hw/boards.h
+++ b/include/hw/boards.h
@@ -8,17 +8,10 @@
 #include "hw/qdev.h"
 #include "qom/object.h"
 
-typedef struct QEMUMachineInitArgs {
-    const MachineClass *machine;
-    ram_addr_t ram_size;
-    const char *boot_order;
-    const char *kernel_filename;
-    const char *kernel_cmdline;
-    const char *initrd_filename;
-    const char *cpu_model;
-} QEMUMachineInitArgs;
 
-typedef void QEMUMachineInitFunc(QEMUMachineInitArgs *args);
+typedef struct MachineState MachineState;
+
+typedef void QEMUMachineInitFunc(MachineState *ms);
 
 typedef void QEMUMachineResetFunc(void);
 
@@ -62,8 +55,6 @@
 #define MACHINE_CLASS(klass) \
     OBJECT_CLASS_CHECK(MachineClass, (klass), TYPE_MACHINE)
 
-typedef struct MachineState MachineState;
-
 MachineClass *find_default_machine(void);
 extern MachineState *current_machine;
 
@@ -80,7 +71,7 @@
     const char *alias;
     const char *desc;
 
-    void (*init)(QEMUMachineInitArgs *args);
+    void (*init)(MachineState *state);
     void (*reset)(void);
     void (*hot_add_cpu)(const int64_t id, Error **errp);
     int (*kvm_type)(const char *arg);
@@ -112,9 +103,6 @@
     char *accel;
     bool kernel_irqchip;
     int kvm_shadow_mem;
-    char *kernel;
-    char *initrd;
-    char *append;
     char *dtb;
     char *dumpdtb;
     int phandle_start;
@@ -124,7 +112,12 @@
     bool usb;
     char *firmware;
 
-    QEMUMachineInitArgs init_args;
+    ram_addr_t ram_size;
+    const char *boot_order;
+    char *kernel_filename;
+    char *kernel_cmdline;
+    char *initrd_filename;
+    const char *cpu_model;
 };
 
 #endif
diff --git a/include/hw/i386/pc.h b/include/hw/i386/pc.h
index 552fbd8..fa9d997 100644
--- a/include/hw/i386/pc.h
+++ b/include/hw/i386/pc.h
@@ -271,6 +271,10 @@
             .driver   = "apic",\
             .property = "version",\
             .value    = stringify(0x11),\
+        },{\
+            .driver   = "nec-usb-xhci",\
+            .property = "superspeed-ports-first",\
+            .value    = "off",\
         },\
         {\
             .driver   = "pci-serial",\
diff --git a/include/hw/input/hid.h b/include/hw/input/hid.h
index 2567879..2127c7c 100644
--- a/include/hw/input/hid.h
+++ b/include/hw/input/hid.h
@@ -2,6 +2,7 @@
 #define QEMU_HID_H
 
 #include "migration/vmstate.h"
+#include "ui/input.h"
 
 #define HID_MOUSE     1
 #define HID_TABLET    2
@@ -22,7 +23,6 @@
 typedef struct HIDMouseState {
     HIDPointerEvent queue[QUEUE_LENGTH];
     int mouse_grabbed;
-    QEMUPutMouseEntry *eh_entry;
 } HIDMouseState;
 
 typedef struct HIDKeyboardState {
@@ -31,7 +31,6 @@
     uint8_t leds;
     uint8_t key[16];
     int32_t keys;
-    QEMUPutKbdEntry *eh_entry;
 } HIDKeyboardState;
 
 struct HIDState {
@@ -47,6 +46,7 @@
     bool idle_pending;
     QEMUTimer *idle_timer;
     HIDEventFunc event;
+    QemuInputHandlerState *s;
 };
 
 void hid_init(HIDState *hs, int kind, HIDEventFunc event);
diff --git a/include/hw/qdev-core.h b/include/hw/qdev-core.h
index bbed829..9221cfc 100644
--- a/include/hw/qdev-core.h
+++ b/include/hw/qdev-core.h
@@ -131,6 +131,17 @@
     const char *bus_type;
 } DeviceClass;
 
+typedef struct NamedGPIOList NamedGPIOList;
+
+struct NamedGPIOList {
+    char *name;
+    qemu_irq *in;
+    int num_in;
+    qemu_irq *out;
+    int num_out;
+    QLIST_ENTRY(NamedGPIOList) node;
+};
+
 /**
  * DeviceState:
  * @realized: Indicates whether the device has been fully constructed.
@@ -148,10 +159,7 @@
     QemuOpts *opts;
     int hotplugged;
     BusState *parent_bus;
-    int num_gpio_out;
-    qemu_irq *gpio_out;
-    int num_gpio_in;
-    qemu_irq *gpio_in;
+    QLIST_HEAD(, NamedGPIOList) gpios;
     QLIST_HEAD(, BusState) child_bus;
     int num_child_bus;
     int instance_id_alias;
@@ -260,7 +268,11 @@
 bool qdev_machine_modified(void);
 
 qemu_irq qdev_get_gpio_in(DeviceState *dev, int n);
+qemu_irq qdev_get_gpio_in_named(DeviceState *dev, const char *name, int n);
+
 void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
+void qdev_connect_gpio_out_named(DeviceState *dev, const char *name, int n,
+                                 qemu_irq pin);
 
 BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
 
@@ -270,6 +282,10 @@
 /* GPIO inputs also double as IRQ sinks.  */
 void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n);
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n);
+void qdev_init_gpio_in_named(DeviceState *dev, qemu_irq_handler handler,
+                             const char *name, int n);
+void qdev_init_gpio_out_named(DeviceState *dev, qemu_irq *pins,
+                              const char *name, int n);
 
 BusState *qdev_get_parent_bus(DeviceState *dev);
 
diff --git a/include/hw/s390x/adapter.h b/include/hw/s390x/adapter.h
new file mode 100644
index 0000000..7f17035
--- /dev/null
+++ b/include/hw/s390x/adapter.h
@@ -0,0 +1,23 @@
+/*
+ * s390 adapter definitions
+ *
+ * Copyright 2013,2014 IBM Corp.
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef S390X_ADAPTER_H
+#define S390X_ADAPTER_H
+
+struct AdapterInfo {
+    uint64_t ind_addr;
+    uint64_t summary_addr;
+    uint64_t ind_offset;
+    uint32_t summary_offset;
+    uint32_t adapter_id;
+};
+
+#endif
diff --git a/include/hw/s390x/s390_flic.h b/include/hw/s390x/s390_flic.h
index 497b219..489d73b 100644
--- a/include/hw/s390x/s390_flic.h
+++ b/include/hw/s390x/s390_flic.h
@@ -1,33 +1,76 @@
 /*
- * QEMU S390x KVM floating interrupt controller (flic)
+ * QEMU S390x floating interrupt controller (flic)
  *
  * Copyright 2014 IBM Corp.
  * Author(s): Jens Freimann <jfrei@linux.vnet.ibm.com>
+ *            Cornelia Huck <cornelia.huck@de.ibm.com>
  *
  * This work is licensed under the terms of the GNU GPL, version 2 or (at
  * your option) any later version. See the COPYING file in the top-level
  * directory.
  */
 
-#ifndef __KVM_S390_FLIC_H
-#define __KVM_S390_FLIC_H
+#ifndef __HW_S390_FLIC_H
+#define __HW_S390_FLIC_H
 
 #include "hw/sysbus.h"
+#include "hw/s390x/adapter.h"
+#include "hw/virtio/virtio.h"
 
-#define TYPE_KVM_S390_FLIC "s390-flic"
+typedef struct AdapterRoutes {
+    AdapterInfo adapter;
+    int num_routes;
+    int gsi[VIRTIO_PCI_QUEUE_MAX];
+} AdapterRoutes;
+
+#define TYPE_S390_FLIC_COMMON "s390-flic"
+#define S390_FLIC_COMMON(obj) \
+    OBJECT_CHECK(S390FLICState, (obj), TYPE_S390_FLIC_COMMON)
+
+typedef struct S390FLICState {
+    SysBusDevice parent_obj;
+
+} S390FLICState;
+
+#define S390_FLIC_COMMON_CLASS(klass) \
+    OBJECT_CLASS_CHECK(S390FLICStateClass, (klass), TYPE_S390_FLIC_COMMON)
+#define S390_FLIC_COMMON_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(S390FLICStateClass, (obj), TYPE_S390_FLIC_COMMON)
+
+typedef struct S390FLICStateClass {
+    DeviceClass parent_class;
+
+    int (*register_io_adapter)(S390FLICState *fs, uint32_t id, uint8_t isc,
+                               bool swap, bool maskable);
+    int (*io_adapter_map)(S390FLICState *fs, uint32_t id, uint64_t map_addr,
+                          bool do_map);
+    int (*add_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
+    void (*release_adapter_routes)(S390FLICState *fs, AdapterRoutes *routes);
+} S390FLICStateClass;
+
+#define TYPE_KVM_S390_FLIC "s390-flic-kvm"
 #define KVM_S390_FLIC(obj) \
     OBJECT_CHECK(KVMS390FLICState, (obj), TYPE_KVM_S390_FLIC)
 
-typedef struct KVMS390FLICState {
-    SysBusDevice parent_obj;
+#define TYPE_QEMU_S390_FLIC "s390-flic-qemu"
+#define QEMU_S390_FLIC(obj) \
+    OBJECT_CHECK(QEMUS390FLICState, (obj), TYPE_QEMU_S390_FLIC)
 
-    uint32_t fd;
-} KVMS390FLICState;
+typedef struct QEMUS390FLICState {
+    S390FLICState parent_obj;
+} QEMUS390FLICState;
+
+void s390_flic_init(void);
+
+S390FLICState *s390_get_flic(void);
 
 #ifdef CONFIG_KVM
-void s390_flic_init(void);
+DeviceState *s390_flic_kvm_create(void);
 #else
-static inline void s390_flic_init(void) { }
+static inline DeviceState *s390_flic_kvm_create(void)
+{
+    return NULL;
+}
 #endif
 
-#endif /* __KVM_S390_FLIC_H */
+#endif /* __HW_S390_FLIC_H */
diff --git a/include/hw/ssi.h b/include/hw/ssi.h
index 6c13fb2..df0f838 100644
--- a/include/hw/ssi.h
+++ b/include/hw/ssi.h
@@ -23,6 +23,8 @@
 #define SSI_SLAVE_GET_CLASS(obj) \
      OBJECT_GET_CLASS(SSISlaveClass, (obj), TYPE_SSI_SLAVE)
 
+#define SSI_GPIO_CS "ssi-gpio-cs"
+
 typedef enum {
     SSI_CS_NONE = 0,
     SSI_CS_LOW,
diff --git a/include/hw/usb.h b/include/hw/usb.h
index 1919bdc..8bcab48 100644
--- a/include/hw/usb.h
+++ b/include/hw/usb.h
@@ -458,6 +458,7 @@
 void usb_combined_input_packet_complete(USBDevice *dev, USBPacket *p);
 void usb_combined_packet_cancel(USBDevice *dev, USBPacket *p);
 
+void usb_pick_speed(USBPort *port);
 void usb_attach(USBPort *port);
 void usb_detach(USBPort *port);
 void usb_port_reset(USBPort *port);
diff --git a/include/hw/usb/ehci-regs.h b/include/hw/usb/ehci-regs.h
new file mode 100644
index 0000000..616f1b8
--- /dev/null
+++ b/include/hw/usb/ehci-regs.h
@@ -0,0 +1,82 @@
+#ifndef HW_USB_EHCI_REGS_H
+#define HW_USB_EHCI_REGS_H 1
+
+/* Capability Registers Base Address - section 2.2 */
+#define CAPLENGTH        0x0000  /* 1-byte, 0x0001 reserved */
+#define HCIVERSION       0x0002  /* 2-bytes, i/f version # */
+#define HCSPARAMS        0x0004  /* 4-bytes, structural params */
+#define HCCPARAMS        0x0008  /* 4-bytes, capability params */
+#define EECP             HCCPARAMS + 1
+#define HCSPPORTROUTE1   0x000c
+#define HCSPPORTROUTE2   0x0010
+
+#define USBCMD           0x0000
+#define USBCMD_RUNSTOP   (1 << 0)      // run / Stop
+#define USBCMD_HCRESET   (1 << 1)      // HC Reset
+#define USBCMD_FLS       (3 << 2)      // Frame List Size
+#define USBCMD_FLS_SH    2             // Frame List Size Shift
+#define USBCMD_PSE       (1 << 4)      // Periodic Schedule Enable
+#define USBCMD_ASE       (1 << 5)      // Asynch Schedule Enable
+#define USBCMD_IAAD      (1 << 6)      // Int Asynch Advance Doorbell
+#define USBCMD_LHCR      (1 << 7)      // Light Host Controller Reset
+#define USBCMD_ASPMC     (3 << 8)      // Async Sched Park Mode Count
+#define USBCMD_ASPME     (1 << 11)     // Async Sched Park Mode Enable
+#define USBCMD_ITC       (0x7f << 16)  // Int Threshold Control
+#define USBCMD_ITC_SH    16            // Int Threshold Control Shift
+
+#define USBSTS           0x0004
+#define USBSTS_RO_MASK   0x0000003f
+#define USBSTS_INT       (1 << 0)      // USB Interrupt
+#define USBSTS_ERRINT    (1 << 1)      // Error Interrupt
+#define USBSTS_PCD       (1 << 2)      // Port Change Detect
+#define USBSTS_FLR       (1 << 3)      // Frame List Rollover
+#define USBSTS_HSE       (1 << 4)      // Host System Error
+#define USBSTS_IAA       (1 << 5)      // Interrupt on Async Advance
+#define USBSTS_HALT      (1 << 12)     // HC Halted
+#define USBSTS_REC       (1 << 13)     // Reclamation
+#define USBSTS_PSS       (1 << 14)     // Periodic Schedule Status
+#define USBSTS_ASS       (1 << 15)     // Asynchronous Schedule Status
+
+/*
+ *  Interrupt enable bits correspond to the interrupt active bits in USBSTS
+ *  so no need to redefine here.
+ */
+#define USBINTR              0x0008
+#define USBINTR_MASK         0x0000003f
+
+#define FRINDEX              0x000c
+#define CTRLDSSEGMENT        0x0010
+#define PERIODICLISTBASE     0x0014
+#define ASYNCLISTADDR        0x0018
+#define ASYNCLISTADDR_MASK   0xffffffe0
+
+#define CONFIGFLAG           0x0040
+
+/*
+ * Bits that are reserved or are read-only are masked out of values
+ * written to us by software
+ */
+#define PORTSC_RO_MASK       0x007001c0
+#define PORTSC_RWC_MASK      0x0000002a
+#define PORTSC_WKOC_E        (1 << 22)    // Wake on Over Current Enable
+#define PORTSC_WKDS_E        (1 << 21)    // Wake on Disconnect Enable
+#define PORTSC_WKCN_E        (1 << 20)    // Wake on Connect Enable
+#define PORTSC_PTC           (15 << 16)   // Port Test Control
+#define PORTSC_PTC_SH        16           // Port Test Control shift
+#define PORTSC_PIC           (3 << 14)    // Port Indicator Control
+#define PORTSC_PIC_SH        14           // Port Indicator Control Shift
+#define PORTSC_POWNER        (1 << 13)    // Port Owner
+#define PORTSC_PPOWER        (1 << 12)    // Port Power
+#define PORTSC_LINESTAT      (3 << 10)    // Port Line Status
+#define PORTSC_LINESTAT_SH   10           // Port Line Status Shift
+#define PORTSC_PRESET        (1 << 8)     // Port Reset
+#define PORTSC_SUSPEND       (1 << 7)     // Port Suspend
+#define PORTSC_FPRES         (1 << 6)     // Force Port Resume
+#define PORTSC_OCC           (1 << 5)     // Over Current Change
+#define PORTSC_OCA           (1 << 4)     // Over Current Active
+#define PORTSC_PEDC          (1 << 3)     // Port Enable/Disable Change
+#define PORTSC_PED           (1 << 2)     // Port Enable/Disable
+#define PORTSC_CSC           (1 << 1)     // Connect Status Change
+#define PORTSC_CONNECT       (1 << 0)     // Current Connect Status
+
+#endif /* HW_USB_EHCI_REGS_H */
diff --git a/include/hw/usb/uhci-regs.h b/include/hw/usb/uhci-regs.h
new file mode 100644
index 0000000..c7315c5
--- /dev/null
+++ b/include/hw/usb/uhci-regs.h
@@ -0,0 +1,40 @@
+#ifndef HW_USB_UHCI_REGS_H
+#define HW_USB_UHCI_REGS_H 1
+
+#define UHCI_CMD_FGR      (1 << 4)
+#define UHCI_CMD_EGSM     (1 << 3)
+#define UHCI_CMD_GRESET   (1 << 2)
+#define UHCI_CMD_HCRESET  (1 << 1)
+#define UHCI_CMD_RS       (1 << 0)
+
+#define UHCI_STS_HCHALTED (1 << 5)
+#define UHCI_STS_HCPERR   (1 << 4)
+#define UHCI_STS_HSERR    (1 << 3)
+#define UHCI_STS_RD       (1 << 2)
+#define UHCI_STS_USBERR   (1 << 1)
+#define UHCI_STS_USBINT   (1 << 0)
+
+#define TD_CTRL_SPD     (1 << 29)
+#define TD_CTRL_ERROR_SHIFT  27
+#define TD_CTRL_IOS     (1 << 25)
+#define TD_CTRL_IOC     (1 << 24)
+#define TD_CTRL_ACTIVE  (1 << 23)
+#define TD_CTRL_STALL   (1 << 22)
+#define TD_CTRL_BABBLE  (1 << 20)
+#define TD_CTRL_NAK     (1 << 19)
+#define TD_CTRL_TIMEOUT (1 << 18)
+
+#define UHCI_PORT_SUSPEND (1 << 12)
+#define UHCI_PORT_RESET (1 << 9)
+#define UHCI_PORT_LSDA  (1 << 8)
+#define UHCI_PORT_RSVD1 (1 << 7)
+#define UHCI_PORT_RD    (1 << 6)
+#define UHCI_PORT_ENC   (1 << 3)
+#define UHCI_PORT_EN    (1 << 2)
+#define UHCI_PORT_CSC   (1 << 1)
+#define UHCI_PORT_CCS   (1 << 0)
+
+#define UHCI_PORT_READ_ONLY    (0x1bb)
+#define UHCI_PORT_WRITE_CLEAR  (UHCI_PORT_CSC | UHCI_PORT_ENC)
+
+#endif /* HW_USB_UHCI_REGS_H */
diff --git a/include/qapi/error.h b/include/qapi/error.h
index 7995801..d712089 100644
--- a/include/qapi/error.h
+++ b/include/qapi/error.h
@@ -67,12 +67,6 @@
  */
 void error_setg_file_open(Error **errp, int os_errno, const char *filename);
 
-/**
- * Returns true if an indirect pointer to an error is pointing to a valid
- * error object.
- */
-bool error_is_set(Error **errp);
-
 /*
  * Get the error class of an error object.
  */
diff --git a/include/qapi/qmp/qdict.h b/include/qapi/qmp/qdict.h
index 1ddf97b..d68f4eb 100644
--- a/include/qapi/qmp/qdict.h
+++ b/include/qapi/qmp/qdict.h
@@ -16,6 +16,7 @@
 #include "qapi/qmp/qobject.h"
 #include "qapi/qmp/qlist.h"
 #include "qemu/queue.h"
+#include <stdbool.h>
 #include <stdint.h>
 
 #define QDICT_BUCKET_MAX 512
@@ -70,4 +71,6 @@
 void qdict_extract_subqdict(QDict *src, QDict **dst, const char *start);
 void qdict_array_split(QDict *src, QList **dst);
 
+void qdict_join(QDict *dest, QDict *src, bool overwrite);
+
 #endif /* QDICT_H */
diff --git a/include/qemu-common.h b/include/qemu-common.h
index 3f3fd60..66ceceb 100644
--- a/include/qemu-common.h
+++ b/include/qemu-common.h
@@ -318,6 +318,7 @@
 void qemu_iovec_concat_iov(QEMUIOVector *dst,
                            struct iovec *src_iov, unsigned int src_cnt,
                            size_t soffset, size_t sbytes);
+bool qemu_iovec_is_zero(QEMUIOVector *qiov);
 void qemu_iovec_destroy(QEMUIOVector *qiov);
 void qemu_iovec_reset(QEMUIOVector *qiov);
 size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
diff --git a/include/qemu/bswap.h b/include/qemu/bswap.h
index 0f9c6cf..78c1ced 100644
--- a/include/qemu/bswap.h
+++ b/include/qemu/bswap.h
@@ -215,9 +215,10 @@
  *   q: 64 bits
  *
  * endian is:
- * (empty): host endian
+ *   he   : host endian
  *   be   : big endian
  *   le   : little endian
+ * (except for byte accesses, which have no endian infix).
  */
 
 static inline int ldub_p(const void *ptr)
@@ -239,82 +240,82 @@
    operations.  Thus we don't need to play games with packed attributes, or
    inline byte-by-byte stores.  */
 
-static inline int lduw_p(const void *ptr)
+static inline int lduw_he_p(const void *ptr)
 {
     uint16_t r;
     memcpy(&r, ptr, sizeof(r));
     return r;
 }
 
-static inline int ldsw_p(const void *ptr)
+static inline int ldsw_he_p(const void *ptr)
 {
     int16_t r;
     memcpy(&r, ptr, sizeof(r));
     return r;
 }
 
-static inline void stw_p(void *ptr, uint16_t v)
+static inline void stw_he_p(void *ptr, uint16_t v)
 {
     memcpy(ptr, &v, sizeof(v));
 }
 
-static inline int ldl_p(const void *ptr)
+static inline int ldl_he_p(const void *ptr)
 {
     int32_t r;
     memcpy(&r, ptr, sizeof(r));
     return r;
 }
 
-static inline void stl_p(void *ptr, uint32_t v)
+static inline void stl_he_p(void *ptr, uint32_t v)
 {
     memcpy(ptr, &v, sizeof(v));
 }
 
-static inline uint64_t ldq_p(const void *ptr)
+static inline uint64_t ldq_he_p(const void *ptr)
 {
     uint64_t r;
     memcpy(&r, ptr, sizeof(r));
     return r;
 }
 
-static inline void stq_p(void *ptr, uint64_t v)
+static inline void stq_he_p(void *ptr, uint64_t v)
 {
     memcpy(ptr, &v, sizeof(v));
 }
 
 static inline int lduw_le_p(const void *ptr)
 {
-    return (uint16_t)le_bswap(lduw_p(ptr), 16);
+    return (uint16_t)le_bswap(lduw_he_p(ptr), 16);
 }
 
 static inline int ldsw_le_p(const void *ptr)
 {
-    return (int16_t)le_bswap(lduw_p(ptr), 16);
+    return (int16_t)le_bswap(lduw_he_p(ptr), 16);
 }
 
 static inline int ldl_le_p(const void *ptr)
 {
-    return le_bswap(ldl_p(ptr), 32);
+    return le_bswap(ldl_he_p(ptr), 32);
 }
 
 static inline uint64_t ldq_le_p(const void *ptr)
 {
-    return le_bswap(ldq_p(ptr), 64);
+    return le_bswap(ldq_he_p(ptr), 64);
 }
 
 static inline void stw_le_p(void *ptr, uint16_t v)
 {
-    stw_p(ptr, le_bswap(v, 16));
+    stw_he_p(ptr, le_bswap(v, 16));
 }
 
 static inline void stl_le_p(void *ptr, uint32_t v)
 {
-    stl_p(ptr, le_bswap(v, 32));
+    stl_he_p(ptr, le_bswap(v, 32));
 }
 
 static inline void stq_le_p(void *ptr, uint64_t v)
 {
-    stq_p(ptr, le_bswap(v, 64));
+    stq_he_p(ptr, le_bswap(v, 64));
 }
 
 /* float access */
@@ -349,37 +350,37 @@
 
 static inline int lduw_be_p(const void *ptr)
 {
-    return (uint16_t)be_bswap(lduw_p(ptr), 16);
+    return (uint16_t)be_bswap(lduw_he_p(ptr), 16);
 }
 
 static inline int ldsw_be_p(const void *ptr)
 {
-    return (int16_t)be_bswap(lduw_p(ptr), 16);
+    return (int16_t)be_bswap(lduw_he_p(ptr), 16);
 }
 
 static inline int ldl_be_p(const void *ptr)
 {
-    return be_bswap(ldl_p(ptr), 32);
+    return be_bswap(ldl_he_p(ptr), 32);
 }
 
 static inline uint64_t ldq_be_p(const void *ptr)
 {
-    return be_bswap(ldq_p(ptr), 64);
+    return be_bswap(ldq_he_p(ptr), 64);
 }
 
 static inline void stw_be_p(void *ptr, uint16_t v)
 {
-    stw_p(ptr, be_bswap(v, 16));
+    stw_he_p(ptr, be_bswap(v, 16));
 }
 
 static inline void stl_be_p(void *ptr, uint32_t v)
 {
-    stl_p(ptr, be_bswap(v, 32));
+    stl_he_p(ptr, be_bswap(v, 32));
 }
 
 static inline void stq_be_p(void *ptr, uint64_t v)
 {
-    stq_p(ptr, be_bswap(v, 64));
+    stq_he_p(ptr, be_bswap(v, 64));
 }
 
 /* float access */
diff --git a/include/qemu/int128.h b/include/qemu/int128.h
index f597031..fb782aa 100644
--- a/include/qemu/int128.h
+++ b/include/qemu/int128.h
@@ -38,6 +38,11 @@
     return (Int128) { 0, 1 };
 }
 
+static inline Int128 int128_exts64(int64_t a)
+{
+    return (Int128) { .lo = a, .hi = (a < 0) ? -1 : 0 };
+}
+
 static inline Int128 int128_and(Int128 a, Int128 b)
 {
     return (Int128) { a.lo & b.lo, a.hi & b.hi };
diff --git a/include/qemu/typedefs.h b/include/qemu/typedefs.h
index 86bab12..5f20b0e 100644
--- a/include/qemu/typedefs.h
+++ b/include/qemu/typedefs.h
@@ -74,5 +74,6 @@
 typedef struct FWCfgState FWCfgState;
 typedef struct PcGuestInfo PcGuestInfo;
 typedef struct Range Range;
+typedef struct AdapterInfo AdapterInfo;
 
 #endif /* QEMU_TYPEDEFS_H */
diff --git a/include/qom/cpu.h b/include/qom/cpu.h
index df977c8..4b352a2 100644
--- a/include/qom/cpu.h
+++ b/include/qom/cpu.h
@@ -80,6 +80,8 @@
  * @has_work: Callback for checking if there is work to do.
  * @do_interrupt: Callback for interrupt handling.
  * @do_unassigned_access: Callback for unassigned access handling.
+ * @do_unaligned_access: Callback for unaligned access handling, if
+ * the target defines #ALIGNED_ONLY.
  * @memory_rw_debug: Callback for GDB memory access.
  * @dump_state: Callback for dumping state.
  * @dump_statistics: Callback for dumping statistics.
@@ -112,6 +114,8 @@
     bool (*has_work)(CPUState *cpu);
     void (*do_interrupt)(CPUState *cpu);
     CPUUnassignedAccess do_unassigned_access;
+    void (*do_unaligned_access)(CPUState *cpu, vaddr addr,
+                                int is_write, int is_user, uintptr_t retaddr);
     int (*memory_rw_debug)(CPUState *cpu, vaddr addr,
                            uint8_t *buf, int len, bool is_write);
     void (*dump_state)(CPUState *cpu, FILE *f, fprintf_function cpu_fprintf,
@@ -544,8 +548,7 @@
 
 #endif /* USER_ONLY */
 
-#ifndef CONFIG_USER_ONLY
-
+#ifdef CONFIG_SOFTMMU
 static inline void cpu_unassigned_access(CPUState *cpu, hwaddr addr,
                                          bool is_write, bool is_exec,
                                          int opaque, unsigned size)
@@ -557,6 +560,14 @@
     }
 }
 
+static inline void cpu_unaligned_access(CPUState *cpu, vaddr addr,
+                                        int is_write, int is_user,
+                                        uintptr_t retaddr)
+{
+    CPUClass *cc = CPU_GET_CLASS(cpu);
+
+    return cc->do_unaligned_access(cpu, addr, is_write, is_user, retaddr);
+}
 #endif
 
 /**
diff --git a/include/sysemu/kvm.h b/include/sysemu/kvm.h
index e7ad9d1..e79e92c 100644
--- a/include/sysemu/kvm.h
+++ b/include/sysemu/kvm.h
@@ -300,7 +300,7 @@
         };                                                           \
         uint64_t args_tmp[] = { __VA_ARGS__ };                       \
         int i;                                                       \
-        for (i = 0; i < ARRAY_SIZE(args_tmp) &&                      \
+        for (i = 0; i < (int)ARRAY_SIZE(args_tmp) &&                 \
                      i < ARRAY_SIZE(cap.args); i++) {                \
             cap.args[i] = args_tmp[i];                               \
         }                                                            \
@@ -315,7 +315,7 @@
         };                                                           \
         uint64_t args_tmp[] = { __VA_ARGS__ };                       \
         int i;                                                       \
-        for (i = 0; i < ARRAY_SIZE(args_tmp) &&                      \
+        for (i = 0; i < (int)ARRAY_SIZE(args_tmp) &&                 \
                      i < ARRAY_SIZE(cap.args); i++) {                \
             cap.args[i] = args_tmp[i];                               \
         }                                                            \
@@ -363,6 +363,8 @@
 int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
+int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter);
+
 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
                                    EventNotifier *rn, int virq);
 int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
diff --git a/include/ui/console.h b/include/ui/console.h
index 8a86617..edbaa9b 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -81,6 +81,9 @@
 #define QEMU_KEY_CTRL_PAGEUP     0xe406
 #define QEMU_KEY_CTRL_PAGEDOWN   0xe407
 
+void kbd_put_keysym_console(QemuConsole *s, int keysym);
+bool kbd_put_qcode_console(QemuConsole *s, int qcode);
+void kbd_put_string_console(QemuConsole *s, const char *str, int len);
 void kbd_put_keysym(int keysym);
 
 /* consoles */
diff --git a/include/ui/input.h b/include/ui/input.h
index 3d3d487..5d5ac00 100644
--- a/include/ui/input.h
+++ b/include/ui/input.h
@@ -29,6 +29,9 @@
 void qemu_input_handler_activate(QemuInputHandlerState *s);
 void qemu_input_handler_deactivate(QemuInputHandlerState *s);
 void qemu_input_handler_unregister(QemuInputHandlerState *s);
+void qemu_input_handler_bind(QemuInputHandlerState *s,
+                             const char *device_id, int head,
+                             Error **errp);
 void qemu_input_event_send(QemuConsole *src, InputEvent *evt);
 void qemu_input_event_sync(void);
 
@@ -36,6 +39,8 @@
 void qemu_input_event_send_key(QemuConsole *src, KeyValue *key, bool down);
 void qemu_input_event_send_key_number(QemuConsole *src, int num, bool down);
 void qemu_input_event_send_key_qcode(QemuConsole *src, QKeyCode q, bool down);
+void qemu_input_event_send_key_delay(uint32_t delay_ms);
+int qemu_input_key_number_to_qcode(uint8_t nr);
 int qemu_input_key_value_to_number(const KeyValue *value);
 int qemu_input_key_value_to_qcode(const KeyValue *value);
 int qemu_input_key_value_to_scancode(const KeyValue *value, bool down,
diff --git a/iohandler.c b/iohandler.c
index ae2ef8f..cca614f 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -191,6 +191,7 @@
     struct sigaction act;
     sigchld_bh = qemu_bh_new(sigchld_bh_handler, NULL);
 
+    memset(&act, 0, sizeof(act));
     act.sa_handler = sigchld_handler;
     act.sa_flags = SA_NOCLDSTOP;
     sigaction(SIGCHLD, &act, NULL);
diff --git a/kvm-all.c b/kvm-all.c
index a343ede..4e19eff 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -27,6 +27,7 @@
 #include "sysemu/sysemu.h"
 #include "hw/hw.h"
 #include "hw/pci/msi.h"
+#include "hw/s390x/adapter.h"
 #include "exec/gdbstub.h"
 #include "sysemu/kvm.h"
 #include "qemu/bswap.h"
@@ -1236,6 +1237,35 @@
     return kvm_vm_ioctl(s, KVM_IRQFD, &irqfd);
 }
 
+int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
+{
+    struct kvm_irq_routing_entry kroute;
+    int virq;
+
+    if (!kvm_gsi_routing_enabled()) {
+        return -ENOSYS;
+    }
+
+    virq = kvm_irqchip_get_virq(s);
+    if (virq < 0) {
+        return virq;
+    }
+
+    kroute.gsi = virq;
+    kroute.type = KVM_IRQ_ROUTING_S390_ADAPTER;
+    kroute.flags = 0;
+    kroute.u.adapter.summary_addr = adapter->summary_addr;
+    kroute.u.adapter.ind_addr = adapter->ind_addr;
+    kroute.u.adapter.summary_offset = adapter->summary_offset;
+    kroute.u.adapter.ind_offset = adapter->ind_offset;
+    kroute.u.adapter.adapter_id = adapter->adapter_id;
+
+    kvm_add_routing_entry(s, &kroute);
+    kvm_irqchip_commit_routes(s);
+
+    return virq;
+}
+
 #else /* !KVM_CAP_IRQ_ROUTING */
 
 void kvm_init_irq_routing(KVMState *s)
@@ -1256,6 +1286,11 @@
     return -ENOSYS;
 }
 
+int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
+{
+    return -ENOSYS;
+}
+
 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     abort();
@@ -1285,7 +1320,8 @@
     int ret;
 
     if (!qemu_opt_get_bool(qemu_get_machine_opts(), "kernel_irqchip", true) ||
-        !kvm_check_extension(s, KVM_CAP_IRQCHIP)) {
+        (!kvm_check_extension(s, KVM_CAP_IRQCHIP) &&
+         (kvm_vm_enable_cap(s, KVM_CAP_S390_IRQCHIP, 0) < 0))) {
         return 0;
     }
 
@@ -1374,7 +1410,7 @@
 
     ret = kvm_ioctl(s, KVM_GET_API_VERSION, 0);
     if (ret < KVM_API_VERSION) {
-        if (ret > 0) {
+        if (ret >= 0) {
             ret = -EINVAL;
         }
         fprintf(stderr, "kvm version too old\n");
@@ -1425,6 +1461,7 @@
     if (mc->kvm_type) {
         type = mc->kvm_type(kvm_type);
     } else if (kvm_type) {
+        ret = -EINVAL;
         fprintf(stderr, "Invalid argument kvm-type=%s\n", kvm_type);
         goto err;
     }
@@ -1525,6 +1562,7 @@
     return 0;
 
 err:
+    assert(ret < 0);
     if (s->vmfd >= 0) {
         close(s->vmfd);
     }
diff --git a/kvm-stub.c b/kvm-stub.c
index 8acda86..ac33d86 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -136,6 +136,11 @@
     return -ENOSYS;
 }
 
+int kvm_irqchip_add_adapter_route(KVMState *s, AdapterInfo *adapter)
+{
+    return -ENOSYS;
+}
+
 int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n,
                                    EventNotifier *rn, int virq)
 {
diff --git a/libcacard/cac.c b/libcacard/cac.c
index 74ef3e3..0a0163d 100644
--- a/libcacard/cac.c
+++ b/libcacard/cac.c
@@ -93,8 +93,8 @@
 static VCardStatus
 cac_applet_pki_reset(VCard *card, int channel)
 {
-    VCardAppletPrivate *applet_private = NULL;
-    CACPKIAppletData *pki_applet = NULL;
+    VCardAppletPrivate *applet_private;
+    CACPKIAppletData *pki_applet;
     applet_private = vcard_get_current_applet_private(card, channel);
     assert(applet_private);
     pki_applet = &(applet_private->u.pki_data);
@@ -113,8 +113,8 @@
 cac_applet_pki_process_apdu(VCard *card, VCardAPDU *apdu,
                             VCardResponse **response)
 {
-    CACPKIAppletData *pki_applet = NULL;
-    VCardAppletPrivate *applet_private = NULL;
+    CACPKIAppletData *pki_applet;
+    VCardAppletPrivate *applet_private;
     int size, next;
     unsigned char *sign_buffer;
     vcard_7816_status_t status;
@@ -169,17 +169,8 @@
         }
         size = apdu->a_Lc;
 
-        sign_buffer = realloc(pki_applet->sign_buffer,
-                      pki_applet->sign_buffer_len+size);
-        if (sign_buffer == NULL) {
-            g_free(pki_applet->sign_buffer);
-            pki_applet->sign_buffer = NULL;
-            pki_applet->sign_buffer_len = 0;
-            *response = vcard_make_response(
-                            VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
-            ret = VCARD_DONE;
-            break;
-        }
+        sign_buffer = g_realloc(pki_applet->sign_buffer,
+                                pki_applet->sign_buffer_len + size);
         memcpy(sign_buffer+pki_applet->sign_buffer_len, apdu->a_body, size);
         size += pki_applet->sign_buffer_len;
         switch (apdu->a_p1) {
@@ -288,7 +279,7 @@
 static void
 cac_delete_pki_applet_private(VCardAppletPrivate *applet_private)
 {
-    CACPKIAppletData *pki_applet_data = NULL;
+    CACPKIAppletData *pki_applet_data;
 
     if (applet_private == NULL) {
         return;
@@ -310,16 +301,11 @@
 cac_new_pki_applet_private(const unsigned char *cert,
                            int cert_len, VCardKey *key)
 {
-    CACPKIAppletData *pki_applet_data = NULL;
-    VCardAppletPrivate *applet_private = NULL;
-    applet_private = (VCardAppletPrivate *)g_malloc(sizeof(VCardAppletPrivate));
+    CACPKIAppletData *pki_applet_data;
+    VCardAppletPrivate *applet_private;
 
+    applet_private = g_new0(VCardAppletPrivate, 1);
     pki_applet_data = &(applet_private->u.pki_data);
-    pki_applet_data->cert_buffer = NULL;
-    pki_applet_data->cert_buffer_len = 0;
-    pki_applet_data->sign_buffer = NULL;
-    pki_applet_data->sign_buffer_len = 0;
-    pki_applet_data->key = NULL;
     pki_applet_data->cert = (unsigned char *)g_malloc(cert_len+1);
     /*
      * if we want to support compression, then we simply change the 0 to a 1
@@ -341,8 +327,8 @@
 cac_new_pki_applet(int i, const unsigned char *cert,
                    int cert_len, VCardKey *key)
 {
-    VCardAppletPrivate *applet_private = NULL;
-    VCardApplet *applet = NULL;
+    VCardAppletPrivate *applet_private;
+    VCardApplet *applet;
     unsigned char pki_aid[] = { 0xa0, 0x00, 0x00, 0x00, 0x79, 0x01, 0x00 };
     int pki_aid_len = sizeof(pki_aid);
 
diff --git a/libcacard/card_7816.c b/libcacard/card_7816.c
index c28bb60..a54f880 100644
--- a/libcacard/card_7816.c
+++ b/libcacard/card_7816.c
@@ -51,7 +51,7 @@
 {
     VCardResponse *new_response;
 
-    new_response = (VCardResponse *)g_malloc(sizeof(VCardResponse));
+    new_response = g_new(VCardResponse, 1);
     new_response->b_data = g_malloc(len + 2);
     memcpy(new_response->b_data, buf, len);
     new_response->b_total_len = len+2;
@@ -132,7 +132,7 @@
 {
     VCardResponse *new_response;
 
-    new_response = (VCardResponse *)g_malloc(sizeof(VCardResponse));
+    new_response = g_new(VCardResponse, 1);
     new_response->b_data = &new_response->b_sw1;
     new_response->b_len = 0;
     new_response->b_total_len = 2;
@@ -149,7 +149,7 @@
 {
     VCardResponse *new_response;
 
-    new_response = (VCardResponse *)g_malloc(sizeof(VCardResponse));
+    new_response = g_new(VCardResponse, 1);
     new_response->b_data = &new_response->b_sw1;
     new_response->b_len = 0;
     new_response->b_total_len = 2;
@@ -336,9 +336,8 @@
         return NULL;
     }
 
-    new_apdu = (VCardAPDU *)g_malloc(sizeof(VCardAPDU));
-    new_apdu->a_data = g_malloc(len);
-    memcpy(new_apdu->a_data, raw_apdu, len);
+    new_apdu = g_new(VCardAPDU, 1);
+    new_apdu->a_data = g_memdup(raw_apdu, len);
     new_apdu->a_len = len;
     *status = vcard_apdu_set_class(new_apdu);
     if (*status != VCARD7816_STATUS_SUCCESS) {
@@ -417,7 +416,7 @@
 VCardResponse *
 vcard_make_response(vcard_7816_status_t status)
 {
-    VCardResponse *response = NULL;
+    VCardResponse *response;
 
     switch (status) {
     /* known 7816 response codes */
@@ -544,9 +543,8 @@
             return VCARD_RESPONSE_GET_STATIC(
                         VCARD7816_STATUS_EXC_ERROR_MEMORY_FAILURE);
         }
+        return response;
     }
-    assert(response);
-    return response;
 }
 
 /*
diff --git a/libcacard/event.c b/libcacard/event.c
index 2d7500f..a2e6c7d 100644
--- a/libcacard/event.c
+++ b/libcacard/event.c
@@ -17,7 +17,7 @@
 {
     VEvent *new_vevent;
 
-    new_vevent = (VEvent *)g_malloc(sizeof(VEvent));
+    new_vevent = g_new(VEvent, 1);
     new_vevent->next = NULL;
     new_vevent->type = type;
     new_vevent->reader = vreader_reference(reader);
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
index 539177b..6aaf085 100644
--- a/libcacard/vcard.c
+++ b/libcacard/vcard.c
@@ -37,9 +37,8 @@
 {
     VCardBufferResponse *new_buffer;
 
-    new_buffer = (VCardBufferResponse *)g_malloc(sizeof(VCardBufferResponse));
-    new_buffer->buffer = (unsigned char *)g_malloc(size);
-    memcpy(new_buffer->buffer, buffer, size);
+    new_buffer = g_new(VCardBufferResponse, 1);
+    new_buffer->buffer = (unsigned char *)g_memdup(buffer, size);
     new_buffer->buffer_len = size;
     new_buffer->current = new_buffer->buffer;
     new_buffer->len = size;
@@ -102,15 +101,11 @@
 {
     VCardApplet *applet;
 
-    applet = (VCardApplet *)g_malloc(sizeof(VCardApplet));
-    applet->next = NULL;
-    applet->applet_private = NULL;
-    applet->applet_private_free = NULL;
+    applet = g_new0(VCardApplet, 1);
     applet->process_apdu = applet_process_function;
     applet->reset_applet = applet_reset_function;
 
-    applet->aid = g_malloc(aid_len);
-    memcpy(applet->aid, aid, aid_len);
+    applet->aid = g_memdup(aid, aid_len);
     applet->aid_len = aid_len;
     return applet;
 }
@@ -149,18 +144,11 @@
 vcard_new(VCardEmul *private, VCardEmulFree private_free)
 {
     VCard *new_card;
-    int i;
 
-    new_card = (VCard *)g_malloc(sizeof(VCard));
-    new_card->applet_list = NULL;
-    for (i = 0; i < MAX_CHANNEL; i++) {
-        new_card->current_applet[i] = NULL;
-    }
-    new_card->vcard_buffer_response = NULL;
+    new_card = g_new0(VCard, 1);
     new_card->type = VCARD_VM;
     new_card->vcard_private = private;
     new_card->vcard_private_free = private_free;
-    new_card->vcard_get_atr = NULL;
     new_card->reference_count = 1;
     return new_card;
 }
@@ -178,8 +166,8 @@
 void
 vcard_free(VCard *vcard)
 {
-    VCardApplet *current_applet = NULL;
-    VCardApplet *next_applet = NULL;
+    VCardApplet *current_applet;
+    VCardApplet *next_applet;
 
     if (vcard == NULL) {
         return;
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
index e2b196d..cefc383 100644
--- a/libcacard/vcard_emul_nss.c
+++ b/libcacard/vcard_emul_nss.c
@@ -94,9 +94,9 @@
 vcard_emul_alloc_arrays(unsigned char ***certsp, int **cert_lenp,
                         VCardKey ***keysp, int cert_count)
 {
-    *certsp = (unsigned char **)g_malloc(sizeof(unsigned char *)*cert_count);
-    *cert_lenp = (int *)g_malloc(sizeof(int)*cert_count);
-    *keysp = (VCardKey **)g_malloc(sizeof(VCardKey *)*cert_count);
+    *certsp = g_new(unsigned char *, cert_count);
+    *cert_lenp = g_new(int, cert_count);
+    *keysp = g_new(VCardKey *, cert_count);
 }
 
 /*
@@ -139,7 +139,7 @@
 {
     VCardKey *key;
 
-    key = (VCardKey *)g_malloc(sizeof(VCardKey));
+    key = g_new(VCardKey, 1);
     key->slot = PK11_ReferenceSlot(slot);
     key->cert = CERT_DupCertificate(cert);
     /* NOTE: if we aren't logged into the token, this could return NULL */
@@ -367,7 +367,7 @@
 vcard_emul_login(VCard *card, unsigned char *pin, int pin_len)
 {
     PK11SlotInfo *slot;
-    unsigned char *pin_string = NULL;
+    unsigned char *pin_string;
     int i;
     SECStatus rv;
 
@@ -423,7 +423,7 @@
 vcard_emul_find_vreader_from_slot(PK11SlotInfo *slot)
 {
     VReaderList *reader_list = vreader_get_reader_list();
-    VReaderListEntry *current_entry = NULL;
+    VReaderListEntry *current_entry;
 
     if (reader_list == NULL) {
         return NULL;
@@ -433,11 +433,13 @@
         VReader *reader = vreader_list_get_reader(current_entry);
         VReaderEmul *reader_emul = vreader_get_private(reader);
         if (reader_emul->slot == slot) {
+            vreader_list_delete(reader_list);
             return reader;
         }
         vreader_free(reader);
     }
 
+    vreader_list_delete(reader_list);
     return NULL;
 }
 
@@ -449,7 +451,7 @@
 {
     VReaderEmul *new_reader_emul;
 
-    new_reader_emul = (VReaderEmul *)g_malloc(sizeof(VReaderEmul));
+    new_reader_emul = g_new(VReaderEmul, 1);
 
     new_reader_emul->slot = PK11_ReferenceSlot(slot);
     new_reader_emul->default_type = type;
@@ -616,11 +618,6 @@
         cert_count++;
     }
 
-    if (cert_count == 0) {
-        PK11_DestroyGenericObjects(firstObj);
-        return NULL;
-    }
-
     /* allocate the arrays */
     vcard_emul_alloc_arrays(&certs, &cert_len, &keys, cert_count);
 
@@ -1050,7 +1047,7 @@
 vcard_emul_replay_insertion_events(void)
 {
     VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry = NULL;
+    VReaderListEntry *next_entry;
     VReaderList *list = vreader_get_reader_list();
 
     for (current_entry = vreader_list_get_first(list); current_entry;
@@ -1059,6 +1056,8 @@
         next_entry = vreader_list_get_next(current_entry);
         vreader_queue_card_event(vreader);
     }
+
+    vreader_list_delete(list);
 }
 
 /*
@@ -1150,7 +1149,7 @@
             char type_str[100];
             VCardEmulType type;
             int count, i;
-            VirtualReaderOptions *vreaderOpt = NULL;
+            VirtualReaderOptions *vreaderOpt;
 
             args = strip(args + 5);
             if (*args != '(') {
@@ -1174,14 +1173,10 @@
 
             if (opts->vreader_count >= reader_count) {
                 reader_count += READER_STEP;
-                vreaderOpt = realloc(opts->vreader,
-                                reader_count * sizeof(*vreaderOpt));
-                if (vreaderOpt == NULL) {
-                    return opts; /* we're done */
-                }
+                opts->vreader = g_renew(VirtualReaderOptions, opts->vreader,
+                                        reader_count);
             }
-            opts->vreader = vreaderOpt;
-            vreaderOpt = &vreaderOpt[opts->vreader_count];
+            vreaderOpt = &opts->vreader[opts->vreader_count];
             vreaderOpt->name = g_strndup(name, name_length);
             vreaderOpt->vname = g_strndup(vname, vname_length);
             vreaderOpt->card_type = type;
@@ -1189,7 +1184,7 @@
                 g_strndup(type_params, type_params_length);
             count = count_tokens(args, ',', ')') + 1;
             vreaderOpt->cert_count = count;
-            vreaderOpt->cert_name = (char **)g_malloc(count*sizeof(char *));
+            vreaderOpt->cert_name = g_new(char *, count);
             for (i = 0; i < count; i++) {
                 const char *cert = args;
                 args = strpbrk(args, ",)");
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
index 7720295..d2a9b7d 100644
--- a/libcacard/vreader.c
+++ b/libcacard/vreader.c
@@ -115,7 +115,7 @@
 {
     VReader *reader;
 
-    reader = (VReader *)g_malloc(sizeof(VReader));
+    reader = g_new(VReader, 1);
     qemu_mutex_init(&reader->lock);
     reader->reference_count = 1;
     reader->name = g_strdup(name);
@@ -283,12 +283,10 @@
                   response->b_sw2, response->b_len, response->b_total_len);
         }
     }
-    assert(card_status == VCARD_DONE);
-    if (card_status == VCARD_DONE) {
-        int size = MIN(*receive_buf_len, response->b_total_len);
-        memcpy(receive_buf, response->b_data, size);
-        *receive_buf_len = size;
-    }
+    assert(card_status == VCARD_DONE && response);
+    int size = MIN(*receive_buf_len, response->b_total_len);
+    memcpy(receive_buf, response->b_data, size);
+    *receive_buf_len = size;
     vcard_response_delete(response);
     vcard_apdu_delete(apdu);
     vcard_free(card); /* free our reference */
@@ -312,10 +310,7 @@
 {
     VReaderListEntry *new_reader_list_entry;
 
-    new_reader_list_entry = (VReaderListEntry *)
-                               g_malloc(sizeof(VReaderListEntry));
-    new_reader_list_entry->next = NULL;
-    new_reader_list_entry->prev = NULL;
+    new_reader_list_entry = g_new0(VReaderListEntry, 1);
     new_reader_list_entry->reader = vreader_reference(reader);
     return new_reader_list_entry;
 }
@@ -336,9 +331,7 @@
 {
     VReaderList *new_reader_list;
 
-    new_reader_list = (VReaderList *)g_malloc(sizeof(VReaderList));
-    new_reader_list->head = NULL;
-    new_reader_list->tail = NULL;
+    new_reader_list = g_new0(VReaderList, 1);
     return new_reader_list;
 }
 
@@ -346,7 +339,7 @@
 vreader_list_delete(VReaderList *list)
 {
     VReaderListEntry *current_entry;
-    VReaderListEntry *next_entry = NULL;
+    VReaderListEntry *next_entry;
     for (current_entry = vreader_list_get_first(list); current_entry;
          current_entry = next_entry) {
         next_entry = vreader_list_get_next(current_entry);
@@ -437,8 +430,8 @@
 static VReaderList *
 vreader_copy_list(VReaderList *list)
 {
-    VReaderList *new_list = NULL;
-    VReaderListEntry *current_entry = NULL;
+    VReaderList *new_list;
+    VReaderListEntry *current_entry;
 
     new_list = vreader_list_new();
     if (new_list == NULL) {
@@ -470,7 +463,7 @@
 vreader_get_reader_by_id(vreader_id_t id)
 {
     VReader *reader = NULL;
-    VReaderListEntry *current_entry = NULL;
+    VReaderListEntry *current_entry;
 
     if (id == (vreader_id_t) -1) {
         return NULL;
@@ -494,7 +487,7 @@
 vreader_get_reader_by_name(const char *name)
 {
     VReader *reader = NULL;
-    VReaderListEntry *current_entry = NULL;
+    VReaderListEntry *current_entry;
 
     vreader_list_lock();
     for (current_entry = vreader_list_get_first(vreader_list); current_entry;
diff --git a/libcacard/vscclient.c b/libcacard/vscclient.c
index 3477ab3..6693900 100644
--- a/libcacard/vscclient.c
+++ b/libcacard/vscclient.c
@@ -131,8 +131,8 @@
 event_thread(void *arg)
 {
     unsigned char atr[MAX_ATR_LEN];
-    int atr_len = MAX_ATR_LEN;
-    VEvent *event = NULL;
+    int atr_len;
+    VEvent *event;
     unsigned int reader_id;
 
 
@@ -502,8 +502,7 @@
             if (reader != NULL) {
                 error = vcard_emul_force_card_insert(reader);
                 printf("insert %s, returned %d\n",
-                       reader ? vreader_get_name(reader)
-                       : "invalid reader", error);
+                       vreader_get_name(reader), error);
             } else {
                 printf("no reader by id %u found\n", reader_id);
             }
@@ -515,8 +514,7 @@
             if (reader != NULL) {
                 error = vcard_emul_force_card_remove(reader);
                 printf("remove %s, returned %d\n",
-                        reader ? vreader_get_name(reader)
-                        : "invalid reader", error);
+                       vreader_get_name(reader), error);
             } else {
                 printf("no reader by id %u found\n", reader_id);
             }
@@ -572,6 +570,7 @@
                        "CARD_PRESENT" : "            ",
                        vreader_get_name(reader));
             }
+            vreader_list_delete(list);
         } else if (*string != 0) {
             printf("valid commands:\n");
             printf("insert [reader_id]\n");
diff --git a/linux-headers/asm-s390/kvm.h b/linux-headers/asm-s390/kvm.h
index c003c6a..98bedf3 100644
--- a/linux-headers/asm-s390/kvm.h
+++ b/linux-headers/asm-s390/kvm.h
@@ -15,6 +15,7 @@
 #include <linux/types.h>
 
 #define __KVM_S390
+#define __KVM_HAVE_GUEST_DEBUG
 
 /* Device control API: s390-specific devices */
 #define KVM_DEV_FLIC_GET_ALL_IRQS	1
@@ -54,6 +55,13 @@
 	__u64 addr;
 };
 
+/* kvm attr_group  on vm fd */
+#define KVM_S390_VM_MEM_CTRL		0
+
+/* kvm attributes for mem_ctrl */
+#define KVM_S390_VM_MEM_ENABLE_CMMA	0
+#define KVM_S390_VM_MEM_CLR_CMMA	1
+
 /* for KVM_GET_REGS and KVM_SET_REGS */
 struct kvm_regs {
 	/* general purpose regs for s390 */
@@ -72,11 +80,31 @@
 	__u64 fprs[16];
 };
 
+#define KVM_GUESTDBG_USE_HW_BP		0x00010000
+
+#define KVM_HW_BP			1
+#define KVM_HW_WP_WRITE			2
+#define KVM_SINGLESTEP			4
+
 struct kvm_debug_exit_arch {
+	__u64 addr;
+	__u8 type;
+	__u8 pad[7]; /* Should be set to 0 */
+};
+
+struct kvm_hw_breakpoint {
+	__u64 addr;
+	__u64 phys_addr;
+	__u64 len;
+	__u8 type;
+	__u8 pad[7]; /* Should be set to 0 */
 };
 
 /* for KVM_SET_GUEST_DEBUG */
 struct kvm_guest_debug_arch {
+	__u32 nr_hw_bp;
+	__u32 pad; /* Should be set to 0 */
+	struct kvm_hw_breakpoint *hw_bp;
 };
 
 #define KVM_SYNC_PREFIX (1UL << 0)
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index b278ab3..42ddc2c 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -416,6 +416,8 @@
 #define KVM_S390_INT_PFAULT_INIT	0xfffe0004u
 #define KVM_S390_INT_PFAULT_DONE	0xfffe0005u
 #define KVM_S390_MCHK			0xfffe1000u
+#define KVM_S390_INT_CLOCK_COMP		0xffff1004u
+#define KVM_S390_INT_CPU_TIMER		0xffff1005u
 #define KVM_S390_INT_VIRTIO		0xffff2603u
 #define KVM_S390_INT_SERVICE		0xffff2401u
 #define KVM_S390_INT_EMERGENCY		0xffff1201u
@@ -515,6 +517,7 @@
 	kvm_ioeventfd_flag_nr_pio,
 	kvm_ioeventfd_flag_nr_deassign,
 	kvm_ioeventfd_flag_nr_virtio_ccw_notify,
+	kvm_ioeventfd_flag_nr_fast_mmio,
 	kvm_ioeventfd_flag_nr_max,
 };
 
@@ -529,7 +532,7 @@
 struct kvm_ioeventfd {
 	__u64 datamatch;
 	__u64 addr;        /* legal pio/mmio address */
-	__u32 len;         /* 1, 2, 4, or 8 bytes    */
+	__u32 len;         /* 1, 2, 4, or 8 bytes; or 0 to ignore length */
 	__s32 fd;
 	__u32 flags;
 	__u8  pad[36];
@@ -743,6 +746,8 @@
 #define KVM_CAP_IOAPIC_POLARITY_IGNORED 97
 #define KVM_CAP_ENABLE_CAP_VM 98
 #define KVM_CAP_S390_IRQCHIP 99
+#define KVM_CAP_IOEVENTFD_NO_LENGTH 100
+#define KVM_CAP_VM_ATTRIBUTES 101
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
diff --git a/linux-user/main.c b/linux-user/main.c
index 882186e..3e21024 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -4052,7 +4052,7 @@
 
 #if defined(TARGET_I386)
     env->cr[0] = CR0_PG_MASK | CR0_WP_MASK | CR0_PE_MASK;
-    env->hflags |= HF_PE_MASK;
+    env->hflags |= HF_PE_MASK | HF_CPL_MASK;
     if (env->features[FEAT_1_EDX] & CPUID_SSE) {
         env->cr[4] |= CR4_OSFXSR_MASK;
         env->hflags |= HF_OSFXSR_MASK;
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 36d4a73..ba3d8ab 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -5,6 +5,7 @@
 #include <string.h>
 
 #include "cpu.h"
+#include "exec/cpu_ldst.h"
 
 #undef DEBUG_REMAP
 #ifdef DEBUG_REMAP
diff --git a/memory.c b/memory.c
index 3f1df23..678661e 100644
--- a/memory.c
+++ b/memory.c
@@ -1722,12 +1722,19 @@
 
 void address_space_destroy(AddressSpace *as)
 {
+    MemoryListener *listener;
+
     /* Flush out anything from MemoryListeners listening in on this */
     memory_region_transaction_begin();
     as->root = NULL;
     memory_region_transaction_commit();
     QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
     address_space_destroy_dispatch(as);
+
+    QTAILQ_FOREACH(listener, &memory_listeners, link) {
+        assert(listener->address_space_filter != as);
+    }
+
     flatview_unref(as->current_map);
     g_free(as->name);
     g_free(as->ioeventfds);
diff --git a/monitor.c b/monitor.c
index 593679a..0565816 100644
--- a/monitor.c
+++ b/monitor.c
@@ -66,6 +66,7 @@
 #include "trace/simple.h"
 #endif
 #include "exec/memory.h"
+#include "exec/cpu_ldst.h"
 #include "qmp-commands.h"
 #include "hmp.h"
 #include "qemu/thread.h"
diff --git a/nbd.c b/nbd.c
index e5084b6..e0d032c 100644
--- a/nbd.c
+++ b/nbd.c
@@ -306,7 +306,7 @@
         [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
         [24 ..  25]   server flags (0)
-        [24 ..  27]   export flags
+        [26 ..  27]   export flags
         [28 .. 151]   reserved     (0)
 
        Negotiation header with options, part 1:
diff --git a/pc-bios/bios-256k.bin b/pc-bios/bios-256k.bin
index 3e60a3d..09686a3 100644
--- a/pc-bios/bios-256k.bin
+++ b/pc-bios/bios-256k.bin
Binary files differ
diff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index d360d0a..2314027 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
Binary files differ
diff --git a/pc-bios/vgabios-cirrus.bin b/pc-bios/vgabios-cirrus.bin
index 2ddef2e..57a5f95 100644
--- a/pc-bios/vgabios-cirrus.bin
+++ b/pc-bios/vgabios-cirrus.bin
Binary files differ
diff --git a/pc-bios/vgabios-qxl.bin b/pc-bios/vgabios-qxl.bin
index f84a984..ed79993 100644
--- a/pc-bios/vgabios-qxl.bin
+++ b/pc-bios/vgabios-qxl.bin
Binary files differ
diff --git a/pc-bios/vgabios-stdvga.bin b/pc-bios/vgabios-stdvga.bin
index 833c464..d3579b4 100644
--- a/pc-bios/vgabios-stdvga.bin
+++ b/pc-bios/vgabios-stdvga.bin
Binary files differ
diff --git a/pc-bios/vgabios-vmware.bin b/pc-bios/vgabios-vmware.bin
index c4bf322..f89845e 100644
--- a/pc-bios/vgabios-vmware.bin
+++ b/pc-bios/vgabios-vmware.bin
Binary files differ
diff --git a/pc-bios/vgabios.bin b/pc-bios/vgabios.bin
index 55c6db3..d3038f4 100644
--- a/pc-bios/vgabios.bin
+++ b/pc-bios/vgabios.bin
Binary files differ
diff --git a/qapi-schema.json b/qapi-schema.json
index 36cb964..7bc33ea 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -691,7 +691,7 @@
 # Information about current migration process.
 #
 # @status: #optional string describing the current migration status.
-#          As of 0.14.0 this can be 'active', 'completed', 'failed' or
+#          As of 0.14.0 this can be 'setup', 'active', 'completed', 'failed' or
 #          'cancelled'. If this field is not returned, no migration process
 #          has been initiated
 #
@@ -942,6 +942,8 @@
 # @encryption_key_missing: true if the backing device is encrypted but an
 #                          valid encryption key is missing
 #
+# @detect_zeroes: detect and optimize zero writes (Since 2.1)
+#
 # @bps: total throughput limit in bytes per second is specified
 #
 # @bps_rd: read throughput limit in bytes per second is specified
@@ -977,6 +979,7 @@
   'data': { 'file': 'str', '*node-name': 'str', 'ro': 'bool', 'drv': 'str',
             '*backing_file': 'str', 'backing_file_depth': 'int',
             'encrypted': 'bool', 'encryption_key_missing': 'bool',
+            'detect_zeroes': 'BlockdevDetectZeroesOptions',
             'bps': 'int', 'bps_rd': 'int', 'bps_wr': 'int',
             'iops': 'int', 'iops_rd': 'int', 'iops_wr': 'int',
             'image': 'ImageInfo',
@@ -4255,6 +4258,22 @@
   'data': [ 'ignore', 'unmap' ] }
 
 ##
+# @BlockdevDetectZeroesOptions
+#
+# Describes the operation mode for the automatic conversion of plain
+# zero writes by the OS to driver specific optimized zero write commands.
+#
+# @off:      Disabled (default)
+# @on:       Enabled
+# @unmap:    Enabled and even try to unmap blocks if possible. This requires
+#            also that @BlockdevDiscardOptions is set to unmap for this device.
+#
+# Since: 2.1
+##
+{ 'enum': 'BlockdevDetectZeroesOptions',
+  'data': [ 'off', 'on', 'unmap' ] }
+
+##
 # @BlockdevAioOptions
 #
 # Selects the AIO backend to handle I/O requests
@@ -4306,20 +4325,22 @@
 # Options that are available for all block devices, independent of the block
 # driver.
 #
-# @driver:      block driver name
-# @id:          #optional id by which the new block device can be referred to.
-#               This is a required option on the top level of blockdev-add, and
-#               currently not allowed on any other level.
-# @node-name:   #optional the name of a block driver state node (Since 2.0)
-# @discard:     #optional discard-related options (default: ignore)
-# @cache:       #optional cache-related options
-# @aio:         #optional AIO backend (default: threads)
-# @rerror:      #optional how to handle read errors on the device
-#               (default: report)
-# @werror:      #optional how to handle write errors on the device
-#               (default: enospc)
-# @read-only:   #optional whether the block device should be read-only
-#               (default: false)
+# @driver:        block driver name
+# @id:            #optional id by which the new block device can be referred to.
+#                 This is a required option on the top level of blockdev-add, and
+#                 currently not allowed on any other level.
+# @node-name:     #optional the name of a block driver state node (Since 2.0)
+# @discard:       #optional discard-related options (default: ignore)
+# @cache:         #optional cache-related options
+# @aio:           #optional AIO backend (default: threads)
+# @rerror:        #optional how to handle read errors on the device
+#                 (default: report)
+# @werror:        #optional how to handle write errors on the device
+#                 (default: enospc)
+# @read-only:     #optional whether the block device should be read-only
+#                 (default: false)
+# @detect-zeroes: #optional detect and optimize zero writes (Since 2.1)
+#                 (default: off)
 #
 # Since: 1.7
 ##
@@ -4332,7 +4353,8 @@
             '*aio': 'BlockdevAioOptions',
             '*rerror': 'BlockdevOnError',
             '*werror': 'BlockdevOnError',
-            '*read-only': 'bool' } }
+            '*read-only': 'bool',
+            '*detect-zeroes': 'BlockdevDetectZeroesOptions' } }
 
 ##
 # @BlockdevOptionsFile
diff --git a/qapi/qmp-output-visitor.c b/qapi/qmp-output-visitor.c
index 74a5684..96b3384 100644
--- a/qapi/qmp-output-visitor.c
+++ b/qapi/qmp-output-visitor.c
@@ -66,6 +66,12 @@
 static QObject *qmp_output_first(QmpOutputVisitor *qov)
 {
     QStackEntry *e = QTAILQ_LAST(&qov->stack, QStack);
+
+    /* FIXME - find a better way to deal with NULL values */
+    if (!e) {
+        return NULL;
+    }
+
     return e->value;
 }
 
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 02cbe43..f87f3d8 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -613,14 +613,20 @@
 {
     ObjectClass *class;
     BusState *child;
+    NamedGPIOList *ngl;
+
     qdev_printf("dev: %s, id \"%s\"\n", object_get_typename(OBJECT(dev)),
                 dev->id ? dev->id : "");
     indent += 2;
-    if (dev->num_gpio_in) {
-        qdev_printf("gpio-in %d\n", dev->num_gpio_in);
-    }
-    if (dev->num_gpio_out) {
-        qdev_printf("gpio-out %d\n", dev->num_gpio_out);
+    QLIST_FOREACH(ngl, &dev->gpios, node) {
+        if (ngl->num_in) {
+            qdev_printf("gpio-in \"%s\" %d\n", ngl->name ? ngl->name : "",
+                        ngl->num_in);
+        }
+        if (ngl->num_out) {
+            qdev_printf("gpio-out \"%s\" %d\n", ngl->name ? ngl->name : "",
+                        ngl->num_out);
+        }
     }
     class = object_get_class(OBJECT(dev));
     do {
diff --git a/qemu-char.c b/qemu-char.c
index 54ed244..17b476e 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -3204,6 +3204,7 @@
                                     void (*init)(struct CharDriverState *s),
                                     Error **errp)
 {
+    Error *local_err = NULL;
     CharDriver *cd;
     CharDriverState *chr;
     GSList *i;
@@ -3245,13 +3246,14 @@
         chr = NULL;
         backend->kind = cd->kind;
         if (cd->parse) {
-            cd->parse(opts, backend, errp);
-            if (error_is_set(errp)) {
+            cd->parse(opts, backend, &local_err);
+            if (local_err) {
+                error_propagate(errp, local_err);
                 goto qapi_out;
             }
         }
         ret = qmp_chardev_add(bid ? bid : id, backend, errp);
-        if (error_is_set(errp)) {
+        if (!ret) {
             goto qapi_out;
         }
 
@@ -3263,7 +3265,7 @@
             backend->kind = CHARDEV_BACKEND_KIND_MUX;
             backend->mux->chardev = g_strdup(bid);
             ret = qmp_chardev_add(id, backend, errp);
-            if (error_is_set(errp)) {
+            if (!ret) {
                 chr = qemu_chr_find(bid);
                 qemu_chr_delete(chr);
                 chr = NULL;
@@ -3620,18 +3622,18 @@
 
 static CharDriverState *qmp_chardev_open_file(ChardevFile *file, Error **errp)
 {
-    int flags, in = -1, out = -1;
+    int flags, in = -1, out;
 
     flags = O_WRONLY | O_TRUNC | O_CREAT | O_BINARY;
     out = qmp_chardev_open_file_source(file->out, flags, errp);
-    if (error_is_set(errp)) {
+    if (out < 0) {
         return NULL;
     }
 
     if (file->has_in) {
         flags = O_RDONLY;
         in = qmp_chardev_open_file_source(file->in, flags, errp);
-        if (error_is_set(errp)) {
+        if (in < 0) {
             qemu_close(out);
             return NULL;
         }
@@ -3647,7 +3649,7 @@
     int fd;
 
     fd = qmp_chardev_open_file_source(serial->device, O_RDWR, errp);
-    if (error_is_set(errp)) {
+    if (fd < 0) {
         return NULL;
     }
     qemu_set_nonblock(fd);
@@ -3665,7 +3667,7 @@
     int fd;
 
     fd = qmp_chardev_open_file_source(parallel->device, O_RDWR, errp);
-    if (error_is_set(errp)) {
+    if (fd < 0) {
         return NULL;
     }
     return qemu_chr_open_pp_fd(fd);
@@ -3692,7 +3694,7 @@
     } else {
         fd = socket_connect(addr, errp, NULL, NULL);
     }
-    if (error_is_set(errp)) {
+    if (fd < 0) {
         return NULL;
     }
     return qemu_chr_open_socket_fd(fd, do_nodelay, is_listen,
@@ -3705,7 +3707,7 @@
     int fd;
 
     fd = socket_dgram(udp->remote, udp->local, errp);
-    if (error_is_set(errp)) {
+    if (fd < 0) {
         return NULL;
     }
     return qemu_chr_open_udp_fd(fd);
@@ -3796,7 +3798,13 @@
         break;
     }
 
-    if (chr == NULL && !error_is_set(errp)) {
+    /*
+     * Character backend open hasn't been fully converted to the Error
+     * API.  Some opens fail without setting an error.  Set a generic
+     * error then.
+     * TODO full conversion to Error API
+     */
+    if (chr == NULL && errp && !*errp) {
         error_setg(errp, "Failed to create chardev");
     }
     if (chr) {
diff --git a/qemu-img.c b/qemu-img.c
index 04ce02a..b3d2bc6 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -70,11 +70,8 @@
 {
     GSequence *seq = opaque;
 
-    if (!g_sequence_lookup(seq, (gpointer)fmt_name,
-                           compare_data, NULL)) {
-        g_sequence_insert_sorted(seq, (gpointer)fmt_name,
-                                 compare_data, NULL);
-    }
+    g_sequence_insert_sorted(seq, (gpointer)fmt_name,
+                             compare_data, NULL);
 }
 
 static void QEMU_NORETURN GCC_FMT_ATTR(1, 2) error_exit(const char *fmt, ...)
@@ -290,6 +287,7 @@
         proto_drv = bdrv_find_protocol(filename, true);
         if (!proto_drv) {
             error_report("Unknown protocol '%s'", filename);
+            free_option_parameters(create_options);
             return 1;
         }
         create_options = append_option_parameters(create_options,
@@ -665,9 +663,7 @@
     ret = collect_image_check(bs, check, filename, fmt, fix);
 
     if (ret == -ENOTSUP) {
-        if (output_format == OFORMAT_HUMAN) {
-            error_report("This image format does not support checks");
-        }
+        error_report("This image format does not support checks");
         ret = 63;
         goto fail;
     }
@@ -1457,7 +1453,7 @@
     ret = bdrv_parse_cache_flags(cache, &flags);
     if (ret < 0) {
         error_report("Invalid cache option: %s", cache);
-        return -1;
+        goto out;
     }
 
     out_bs = bdrv_new_open("target", out_filename, out_fmt, flags, true, quiet);
diff --git a/qemu-io.c b/qemu-io.c
index 9fcd72b..795cf46 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -54,6 +54,7 @@
 
     if (qemuio_bs) {
         fprintf(stderr, "file open already, try 'help close'\n");
+        QDECREF(opts);
         return 1;
     }
 
@@ -61,7 +62,8 @@
         if (bdrv_open(&qemuio_bs, name, NULL, opts, flags | BDRV_O_PROTOCOL,
                       NULL, &local_err))
         {
-            fprintf(stderr, "%s: can't open device %s: %s\n", progname, name,
+            fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
+                    name ? " device " : "", name ?: "",
                     error_get_pretty(local_err));
             error_free(local_err);
             return 1;
@@ -72,7 +74,8 @@
         if (bdrv_open(&qemuio_bs, name, NULL, opts, flags, NULL, &local_err)
             < 0)
         {
-            fprintf(stderr, "%s: can't open device %s: %s\n", progname, name,
+            fprintf(stderr, "%s: can't open%s%s: %s\n", progname,
+                    name ? " device " : "", name ?: "",
                     error_get_pretty(local_err));
             error_free(local_err);
             bdrv_unref(qemuio_bs);
@@ -118,6 +121,7 @@
 
 static QemuOptsList empty_opts = {
     .name = "drive",
+    .merge_lists = true,
     .head = QTAILQ_HEAD_INITIALIZER(empty_opts.head),
     .desc = {
         /* no elements => accept any params */
@@ -132,7 +136,7 @@
     int growable = 0;
     int c;
     QemuOpts *qopts;
-    QDict *opts = NULL;
+    QDict *opts;
 
     while ((c = getopt(argc, argv, "snrgo:")) != EOF) {
         switch (c) {
@@ -149,15 +153,14 @@
             growable = 1;
             break;
         case 'o':
-            qopts = qemu_opts_parse(&empty_opts, optarg, 0);
-            if (qopts == NULL) {
+            if (!qemu_opts_parse(&empty_opts, optarg, 0)) {
                 printf("could not parse option list -- %s\n", optarg);
+                qemu_opts_reset(&empty_opts);
                 return 0;
             }
-            opts = qemu_opts_to_qdict(qopts, opts);
-            qemu_opts_del(qopts);
             break;
         default:
+            qemu_opts_reset(&empty_opts);
             return qemuio_command_usage(&open_cmd);
         }
     }
@@ -166,11 +169,16 @@
         flags |= BDRV_O_RDWR;
     }
 
+    qopts = qemu_opts_find(&empty_opts, NULL);
+    opts = qopts ? qemu_opts_to_qdict(qopts, NULL) : NULL;
+    qemu_opts_reset(&empty_opts);
+
     if (optind == argc - 1) {
         return openfile(argv[optind], flags, growable, opts);
     } else if (optind == argc) {
         return openfile(NULL, flags, growable, opts);
     } else {
+        QDECREF(opts);
         return qemuio_command_usage(&open_cmd);
     }
 }
diff --git a/qemu-nbd.c b/qemu-nbd.c
index eed79fa..ba60436 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -223,7 +223,7 @@
     int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
 
     if (local_err != NULL) {
-        qerror_report_err(local_err);
+        error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
     }
     return fd;
@@ -235,7 +235,7 @@
     int fd = unix_listen(path, NULL, 0, &local_err);
 
     if (local_err != NULL) {
-        qerror_report_err(local_err);
+        error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
     }
     return fd;
@@ -247,7 +247,7 @@
     int fd = unix_connect(path, &local_err);
 
     if (local_err != NULL) {
-        qerror_report_err(local_err);
+        error_report("%s", error_get_pretty(local_err));
         error_free(local_err);
     }
     return fd;
@@ -294,7 +294,7 @@
     fd = open(device, O_RDWR);
     if (fd < 0) {
         /* Linux-only, we can use %m in printf.  */
-        fprintf(stderr, "Failed to open %s: %m", device);
+        fprintf(stderr, "Failed to open %s: %m\n", device);
         goto out_socket;
     }
 
@@ -369,8 +369,10 @@
         return;
     }
 
-    if (fd >= 0 && nbd_client_new(exp, fd, nbd_client_closed)) {
+    if (nbd_client_new(exp, fd, nbd_client_closed)) {
         nb_fds++;
+    } else {
+        close(fd);
     }
 }
 
diff --git a/qemu-nbd.texi b/qemu-nbd.texi
index 0a7e013..46fd483 100644
--- a/qemu-nbd.texi
+++ b/qemu-nbd.texi
@@ -15,7 +15,7 @@
 @item @var{filename}
  is a disk image filename
 @item -p, --port=@var{port}
-  port to listen on (default @samp{1024})
+  port to listen on (default @samp{10809})
 @item -o, --offset=@var{offset}
   offset into the image
 @item -b, --bind=@var{iface}
diff --git a/qemu-options.hx b/qemu-options.hx
index 15779e7..d0714c4 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -414,6 +414,7 @@
     "       [,serial=s][,addr=A][,rerror=ignore|stop|report]\n"
     "       [,werror=ignore|stop|report|enospc][,id=name][,aio=threads|native]\n"
     "       [,readonly=on|off][,copy-on-read=on|off]\n"
+    "       [,detect-zeroes=on|off|unmap]\n"
     "       [[,bps=b]|[[,bps_rd=r][,bps_wr=w]]]\n"
     "       [[,iops=i]|[[,iops_rd=r][,iops_wr=w]]]\n"
     "       [[,bps_max=bm]|[[,bps_rd_max=rm][,bps_wr_max=wm]]]\n"
@@ -475,6 +476,11 @@
 @item copy-on-read=@var{copy-on-read}
 @var{copy-on-read} is "on" or "off" and enables whether to copy read backing
 file sectors into the image file.
+@item detect-zeroes=@var{detect-zeroes}
+@var{detect-zeroes} is "off", "on" or "unmap" and enables the automatic
+conversion of plain zero writes by the OS to driver specific optimized
+zero write commands. You may even choose "unmap" if @var{discard} is set
+to "unmap" to allow a zero write to be converted to an UNMAP operation.
 @end table
 
 By default, the @option{cache=writeback} mode is used. It will report data
@@ -2191,6 +2197,74 @@
 @end example
 
 See also @url{http://www.gluster.org}.
+
+@item HTTP/HTTPS/FTP/FTPS/TFTP
+QEMU supports read-only access to files accessed over http(s), ftp(s) and tftp.
+
+Syntax using a single filename:
+@example
+<protocol>://[<username>[:<password>]@@]<host>/<path>
+@end example
+
+where:
+@table @option
+@item protocol
+'http', 'https', 'ftp', 'ftps', or 'tftp'.
+
+@item username
+Optional username for authentication to the remote server.
+
+@item password
+Optional password for authentication to the remote server.
+
+@item host
+Address of the remote server.
+
+@item path
+Path on the remote server, including any query string.
+@end table
+
+The following options are also supported:
+@table @option
+@item url
+The full URL when passing options to the driver explicitly.
+
+@item readahead
+The amount of data to read ahead with each range request to the remote server.
+This value may optionally have the suffix 'T', 'G', 'M', 'K', 'k' or 'b'. If it
+does not have a suffix, it will be assumed to be in bytes. The value must be a
+multiple of 512 bytes. It defaults to 256k.
+
+@item sslverify
+Whether to verify the remote server's certificate when connecting over SSL. It
+can have the value 'on' or 'off'. It defaults to 'on'.
+@end table
+
+Note that when passing options to qemu explicitly, @option{driver} is the value
+of <protocol>.
+
+Example: boot from a remote Fedora 20 live ISO image
+@example
+qemu-system-x86_64 --drive media=cdrom,file=http://dl.fedoraproject.org/pub/fedora/linux/releases/20/Live/x86_64/Fedora-Live-Desktop-x86_64-20-1.iso,readonly
+
+qemu-system-x86_64 --drive media=cdrom,file.driver=http,file.url=http://dl.fedoraproject.org/pub/fedora/linux/releases/20/Live/x86_64/Fedora-Live-Desktop-x86_64-20-1.iso,readonly
+@end example
+
+Example: boot from a remote Fedora 20 cloud image using a local overlay for
+writes, copy-on-read, and a readahead of 64k
+@example
+qemu-img create -f qcow2 -o backing_file='json:@{"file.driver":"http",, "file.url":"https://dl.fedoraproject.org/pub/fedora/linux/releases/20/Images/x86_64/Fedora-x86_64-20-20131211.1-sda.qcow2",, "file.readahead":"64k"@}' /tmp/Fedora-x86_64-20-20131211.1-sda.qcow2
+
+qemu-system-x86_64 -drive file=/tmp/Fedora-x86_64-20-20131211.1-sda.qcow2,copy-on-read=on
+@end example
+
+Example: boot from an image stored on a VMware vSphere server with a self-signed
+certificate using a local overlay for writes and a readahead of 64k
+@example
+qemu-img create -f qcow2 -o backing_file='json:@{"file.driver":"https",, "file.url":"https://user:password@@vsphere.example.com/folder/test/test-flat.vmdk?dcPath=Datacenter&dsName=datastore1",, "file.sslverify":"off",, "file.readahead":"64k"@}' /tmp/test.qcow2
+
+qemu-system-x86_64 -drive file=/tmp/test.qcow2
+@end example
 ETEXI
 
 STEXI
@@ -2997,7 +3071,8 @@
 Set OpenBIOS nvram @var{variable} to given @var{value} (PPC, SPARC only).
 ETEXI
 DEF("semihosting", 0, QEMU_OPTION_semihosting,
-    "-semihosting    semihosting mode\n", QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA)
+    "-semihosting    semihosting mode\n",
+    QEMU_ARCH_ARM | QEMU_ARCH_M68K | QEMU_ARCH_XTENSA | QEMU_ARCH_LM32)
 STEXI
 @item -semihosting
 @findex -semihosting
diff --git a/qmp-commands.hx b/qmp-commands.hx
index cae890e..d8aa4ed 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -2032,6 +2032,8 @@
          - "iops_rd_max":  read I/O operations max (json-int)
          - "iops_wr_max":  write I/O operations max (json-int)
          - "iops_size": I/O size when limiting by iops (json-int)
+         - "detect_zeroes": detect and optimize zero writing (json-string)
+             - Possible values: "off", "on", "unmap"
          - "image": the detail of the image, it is a json-object containing
             the following:
              - "filename": image file name (json-string)
@@ -2108,6 +2110,7 @@
                "iops_rd_max": 0,
                "iops_wr_max": 0,
                "iops_size": 0,
+               "detect_zeroes": "on",
                "image":{
                   "filename":"disks/test.qcow2",
                   "format":"qcow2",
@@ -2937,7 +2940,7 @@
 The main json-object contains the following:
 
 - "status": migration status (json-string)
-     - Possible values: "active", "completed", "failed", "cancelled"
+     - Possible values: "setup", "active", "completed", "failed", "cancelled"
 - "total-time": total amount of ms since migration started.  If
                 migration has ended, it returns the total migration
                 time (json-int)
diff --git a/qobject/qdict.c b/qobject/qdict.c
index 42ec4c0..ea239f0 100644
--- a/qobject/qdict.c
+++ b/qobject/qdict.c
@@ -665,3 +665,35 @@
         qlist_append_obj(*dst, subqobj ?: QOBJECT(subqdict));
     }
 }
+
+/**
+ * qdict_join(): Absorb the src QDict into the dest QDict, that is, move all
+ * elements from src to dest.
+ *
+ * If an element from src has a key already present in dest, it will not be
+ * moved unless overwrite is true.
+ *
+ * If overwrite is true, the conflicting values in dest will be discarded and
+ * replaced by the corresponding values from src.
+ *
+ * Therefore, with overwrite being true, the src QDict will always be empty when
+ * this function returns. If overwrite is false, the src QDict will be empty
+ * iff there were no conflicts.
+ */
+void qdict_join(QDict *dest, QDict *src, bool overwrite)
+{
+    const QDictEntry *entry, *next;
+
+    entry = qdict_first(src);
+    while (entry) {
+        next = qdict_next(src, entry);
+
+        if (overwrite || !qdict_haskey(dest, entry->key)) {
+            qobject_incref(entry->value);
+            qdict_put_obj(dest, entry->key, entry->value);
+            qdict_del(src, entry->key);
+        }
+
+        entry = next;
+    }
+}
diff --git a/qtest.c b/qtest.c
index 2aba20d..f9695a8 100644
--- a/qtest.c
+++ b/qtest.c
@@ -233,7 +233,8 @@
     g_assert(command);
     if (strcmp(words[0], "irq_intercept_out") == 0
         || strcmp(words[0], "irq_intercept_in") == 0) {
-	DeviceState *dev;
+        DeviceState *dev;
+        NamedGPIOList *ngl;
 
         g_assert(words[1]);
         dev = DEVICE(object_resolve_path(words[1], NULL));
@@ -253,10 +254,18 @@
 	    return;
         }
 
-        if (words[0][14] == 'o') {
-            qemu_irq_intercept_out(&dev->gpio_out, qtest_irq_handler, dev->num_gpio_out);
-        } else {
-            qemu_irq_intercept_in(dev->gpio_in, qtest_irq_handler, dev->num_gpio_in);
+        QLIST_FOREACH(ngl, &dev->gpios, node) {
+            /* We don't support intercept of named GPIOs yet */
+            if (ngl->name) {
+                continue;
+            }
+            if (words[0][14] == 'o') {
+                qemu_irq_intercept_out(&ngl->out, qtest_irq_handler,
+                                       ngl->num_out);
+            } else {
+                qemu_irq_intercept_in(ngl->in, qtest_irq_handler,
+                                      ngl->num_in);
+            }
         }
         irq_intercept_dev = dev;
         qtest_send_prefix(chr);
diff --git a/roms/seabios b/roms/seabios
index b1d4dc9..e51488c 160000
--- a/roms/seabios
+++ b/roms/seabios
@@ -1 +1 @@
-Subproject commit b1d4dc908401719c5de78c25313cf82c7cd1d602
+Subproject commit e51488c5f8800a52ac5c8da7a31b85cca5cc95d2
diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat
index 762544b..d7e97e7 100755
--- a/scripts/kvm/kvm_stat
+++ b/scripts/kvm/kvm_stat
@@ -352,8 +352,8 @@
         return ret
 
 class Stats:
-    def __init__(self, provider, fields = None):
-        self.provider = provider
+    def __init__(self, providers, fields = None):
+        self.providers = providers
         self.fields_filter = fields
         self._update()
     def _update(self):
@@ -362,22 +362,25 @@
             if not self.fields_filter:
                 return True
             return re.match(self.fields_filter, key) is not None
-        self.values = dict([(key, None)
-                            for key in provider.fields()
-                            if wanted(key)])
-        self.provider.select(self.values.keys())
+        self.values = dict()
+        for d in providers:
+            provider_fields = [key for key in d.fields() if wanted(key)]
+            for key in provider_fields:
+                self.values[key] = None
+            d.select(provider_fields)
     def set_fields_filter(self, fields_filter):
         self.fields_filter = fields_filter
         self._update()
     def get(self):
-        new = self.provider.read()
-        for key in self.provider.fields():
-            oldval = self.values.get(key, (0, 0))
-            newval = new[key]
-            newdelta = None
-            if oldval is not None:
-                newdelta = newval - oldval[0]
-            self.values[key] = (newval, newdelta)
+        for d in providers:
+            new = d.read()
+            for key in d.fields():
+                oldval = self.values.get(key, (0, 0))
+                newval = new[key]
+                newdelta = None
+                if oldval is not None:
+                    newdelta = newval - oldval[0]
+                self.values[key] = (newval, newdelta)
         return self.values
 
 if not os.access('/sys/kernel/debug', os.F_OK):
@@ -487,6 +490,18 @@
                    dest = 'log',
                    help = 'run in logging mode (like vmstat)',
                    )
+options.add_option('-t', '--tracepoints',
+                   action = 'store_true',
+                   default = False,
+                   dest = 'tracepoints',
+                   help = 'retrieve statistics from tracepoints',
+                   )
+options.add_option('-d', '--debugfs',
+                   action = 'store_true',
+                   default = False,
+                   dest = 'debugfs',
+                   help = 'retrieve statistics from debugfs',
+                   )
 options.add_option('-f', '--fields',
                    action = 'store',
                    default = None,
@@ -495,12 +510,19 @@
                    )
 (options, args) = options.parse_args(sys.argv)
 
-try:
-    provider = TracepointProvider()
-except:
-    provider = DebugfsProvider()
+providers = []
+if options.tracepoints:
+    providers.append(TracepointProvider())
+if options.debugfs:
+    providers.append(DebugfsProvider())
 
-stats = Stats(provider, fields = options.fields)
+if len(providers) == 0:
+    try:
+        providers = [TracepointProvider()]
+    except:
+        providers = [DebugfsProvider()]
+
+stats = Stats(providers, fields = options.fields)
 
 if options.log:
     log(stats)
diff --git a/scripts/qapi-commands.py b/scripts/qapi-commands.py
index 386f17e..7d93d01 100644
--- a/scripts/qapi-commands.py
+++ b/scripts/qapi-commands.py
@@ -111,7 +111,7 @@
                          argname=c_var(argname), argtype=c_type(argtype))
         else:
             ret += mcgen('''
-%(argtype)s %(argname)s;
+%(argtype)s %(argname)s = {0};
 ''',
                          argname=c_var(argname), argtype=c_type(argtype))
 
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 0265b40..86e9608 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -116,7 +116,7 @@
                     continue
                 try:
                     fobj = open(include_path, 'r')
-                except IOError as e:
+                except IOError, e:
                     raise QAPIExprError(expr_info,
                                         '%s: %s' % (e.strerror, include))
                 exprs_include = QAPISchema(fobj, include, self.include_hist,
diff --git a/include/exec/softmmu_template.h b/softmmu_template.h
similarity index 92%
rename from include/exec/softmmu_template.h
rename to softmmu_template.h
index 73ed7cf..5a07f99 100644
--- a/include/exec/softmmu_template.h
+++ b/softmmu_template.h
@@ -116,6 +116,7 @@
 # define helper_te_st_name  helper_le_st_name
 #endif
 
+#ifndef SOFTMMU_CODE_ACCESS
 static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
                                               hwaddr physaddr,
                                               target_ulong addr,
@@ -135,6 +136,7 @@
     io_mem_read(mr, physaddr, &val, 1 << SHIFT);
     return val;
 }
+#endif
 
 #ifdef SOFTMMU_CODE_ACCESS
 static __attribute__((unused))
@@ -155,7 +157,8 @@
          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                                 mmu_idx, retaddr);
         }
 #endif
         tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
@@ -186,7 +189,8 @@
         unsigned shift;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                             mmu_idx, retaddr);
 #endif
         addr1 = addr & ~(DATA_SIZE - 1);
         addr2 = addr1 + DATA_SIZE;
@@ -204,7 +208,8 @@
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                             mmu_idx, retaddr);
     }
 #endif
 
@@ -237,7 +242,8 @@
          != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                                 mmu_idx, retaddr);
         }
 #endif
         tlb_fill(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
@@ -268,7 +274,8 @@
         unsigned shift;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                             mmu_idx, retaddr);
 #endif
         addr1 = addr & ~(DATA_SIZE - 1);
         addr2 = addr1 + DATA_SIZE;
@@ -286,7 +293,8 @@
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, READ_ACCESS_TYPE,
+                             mmu_idx, retaddr);
     }
 #endif
 
@@ -357,7 +365,7 @@
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
         }
 #endif
         tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
@@ -386,7 +394,7 @@
         int i;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
 #endif
         /* XXX: not efficient, but simple */
         /* Note: relies on the fact that tlb_fill() does not remove the
@@ -405,7 +413,7 @@
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
     }
 #endif
 
@@ -433,7 +441,7 @@
         != (tlb_addr & (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0) {
-            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+            cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
         }
 #endif
         tlb_fill(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
@@ -462,7 +470,7 @@
         int i;
     do_unaligned_access:
 #ifdef ALIGNED_ONLY
-        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
 #endif
         /* XXX: not efficient, but simple */
         /* Note: relies on the fact that tlb_fill() does not remove the
@@ -481,7 +489,7 @@
     /* Handle aligned access or unaligned access in the same page.  */
 #ifdef ALIGNED_ONLY
     if ((addr & (DATA_SIZE - 1)) != 0) {
-        do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
+        cpu_unaligned_access(ENV_GET_CPU(env), addr, 1, mmu_idx, retaddr);
     }
 #endif
 
diff --git a/target-alpha/cpu-qom.h b/target-alpha/cpu-qom.h
index 198f1b1..0caa362 100644
--- a/target-alpha/cpu-qom.h
+++ b/target-alpha/cpu-qom.h
@@ -84,5 +84,7 @@
 hwaddr alpha_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int alpha_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int alpha_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void alpha_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                   int is_write, int is_user, uintptr_t retaddr);
 
 #endif
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index 7ec46b9..2491f0a 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -292,6 +292,7 @@
     cc->handle_mmu_fault = alpha_cpu_handle_mmu_fault;
 #else
     cc->do_unassigned_access = alpha_cpu_unassigned_access;
+    cc->do_unaligned_access = alpha_cpu_do_unaligned_access;
     cc->get_phys_page_debug = alpha_cpu_get_phys_page_debug;
     dc->vmsd = &vmstate_alpha_cpu;
 #endif
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 07d9f63..d9b861f 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -24,6 +24,7 @@
 #include "qemu-common.h"
 
 #define TARGET_LONG_BITS 64
+#define ALIGNED_ONLY
 
 #define CPUArchState struct CPUAlphaState
 
diff --git a/target-alpha/fpu_helper.c b/target-alpha/fpu_helper.c
index ee73155..d2d776c 100644
--- a/target-alpha/fpu_helper.c
+++ b/target-alpha/fpu_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "fpu/softfloat.h"
 
 #define FP_STATUS (env->fp_status)
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index cbd03c4..7c053a3 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -23,7 +23,7 @@
 
 #include "cpu.h"
 #include "fpu/softfloat.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 uint64_t cpu_alpha_load_fpcr (CPUAlphaState *env)
 {
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index 2389e96..a451cfe 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_3(excp, noreturn, env, int, int)
 DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_NO_RWG_SE, i64, env)
 
@@ -121,5 +119,3 @@
 DEF_HELPER_FLAGS_0(get_walltime, TCG_CALL_NO_RWG, i64)
 DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_NO_RWG, void, env, i64)
 #endif
-
-#include "exec/def-helper.h"
diff --git a/target-alpha/int_helper.c b/target-alpha/int_helper.c
index 51ccd41..7a205eb 100644
--- a/target-alpha/int_helper.c
+++ b/target-alpha/int_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 
 
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index 5964bdc..fc4f57a 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -18,8 +18,8 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
-
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 /* Softmmu support */
 #ifndef CONFIG_USER_ONLY
@@ -96,11 +96,11 @@
     return ret;
 }
 
-static void do_unaligned_access(CPUAlphaState *env, target_ulong addr,
-                                int is_write, int is_user, uintptr_t retaddr)
+void alpha_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
+                                   int is_write, int is_user, uintptr_t retaddr)
 {
-    AlphaCPU *cpu = alpha_env_get_cpu(env);
-    CPUState *cs = CPU(cpu);
+    AlphaCPU *cpu = ALPHA_CPU(cs);
+    CPUAlphaState *env = &cpu->env;
     uint64_t pc;
     uint32_t insn;
 
@@ -131,23 +131,6 @@
     dynamic_excp(env, 0, EXCP_MCHK, 0);
 }
 
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-#define ALIGNED_ONLY
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
diff --git a/target-alpha/sys_helper.c b/target-alpha/sys_helper.c
index 187ccf7..ae2e174 100644
--- a/target-alpha/sys_helper.c
+++ b/target-alpha/sys_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "sysemu/sysemu.h"
 #include "qemu/timer.h"
 
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 91c3ed1..cc81e77 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -21,10 +21,10 @@
 #include "disas/disas.h"
 #include "qemu/host-utils.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #undef ALPHA_DEBUG_DISAS
 #define CONFIG_SOFTFLOAT_INLINE
diff --git a/target-arm/arm_ldst.h b/target-arm/arm_ldst.h
new file mode 100644
index 0000000..b1ece01
--- /dev/null
+++ b/target-arm/arm_ldst.h
@@ -0,0 +1,48 @@
+/*
+ * ARM load/store instructions for code (armeb-user support)
+ *
+ *  Copyright (c) 2012 CodeSourcery, LLC
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ARM_LDST_H
+#define ARM_LDST_H
+
+#include "exec/cpu_ldst.h"
+#include "qemu/bswap.h"
+
+/* Load an instruction and return it in the standard little-endian order */
+static inline uint32_t arm_ldl_code(CPUARMState *env, target_ulong addr,
+                                    bool do_swap)
+{
+    uint32_t insn = cpu_ldl_code(env, addr);
+    if (do_swap) {
+        return bswap32(insn);
+    }
+    return insn;
+}
+
+/* Ditto, for a halfword (Thumb) instruction */
+static inline uint16_t arm_lduw_code(CPUARMState *env, target_ulong addr,
+                                     bool do_swap)
+{
+    uint16_t insn = cpu_lduw_code(env, addr);
+    if (do_swap) {
+        return bswap16(insn);
+    }
+    return insn;
+}
+
+#endif
diff --git a/target-arm/cpu.c b/target-arm/cpu.c
index 6c6f2b3..794dcb9 100644
--- a/target-arm/cpu.c
+++ b/target-arm/cpu.c
@@ -370,8 +370,8 @@
 
     init_cpreg_list(cpu);
 
-    cpu_reset(cs);
     qemu_init_vcpu(cs);
+    cpu_reset(cs);
 
     acc->parent_realize(dev, errp);
 }
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index c83f249..7d8332e 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -143,7 +143,7 @@
     uint32_t spsr;
 
     /* Banked registers.  */
-    uint64_t banked_spsr[6];
+    uint64_t banked_spsr[8];
     uint32_t banked_r13[6];
     uint32_t banked_r14[6];
 
@@ -162,8 +162,8 @@
     uint32_t condexec_bits; /* IT bits.  cpsr[15:10,26:25].  */
     uint64_t daif; /* exception masks, in the bits they are in in PSTATE */
 
-    uint64_t elr_el1; /* AArch64 ELR_EL1 */
-    uint64_t sp_el[2]; /* AArch64 banked stack pointers */
+    uint64_t elr_el[4]; /* AArch64 exception link regs  */
+    uint64_t sp_el[4]; /* AArch64 banked stack pointers */
 
     /* System control coprocessor (cp15) */
     struct {
@@ -185,7 +185,7 @@
         uint32_t pmsav5_data_ap; /* PMSAv5 MPU data access permissions */
         uint32_t pmsav5_insn_ap; /* PMSAv5 MPU insn access permissions */
         uint32_t ifsr_el2; /* Fault status registers.  */
-        uint64_t esr_el1;
+        uint64_t esr_el[2];
         uint32_t c6_region[8]; /* MPU base/size registers.  */
         uint64_t far_el1; /* Fault address registers.  */
         uint64_t par_el1;  /* Translation result. */
@@ -198,7 +198,7 @@
         uint32_t c9_pmuserenr; /* perf monitor user enable */
         uint32_t c9_pminten; /* perf monitor interrupt enables */
         uint64_t mair_el1;
-        uint64_t c12_vbar; /* vector base address register */
+        uint64_t vbar_el[4]; /* vector base address register */
         uint32_t c13_fcse; /* FCSE PID.  */
         uint64_t contextidr_el1; /* Context ID.  */
         uint64_t tpidr_el0; /* User RW Thread register.  */
@@ -563,7 +563,9 @@
   ARM_CPU_MODE_FIQ = 0x11,
   ARM_CPU_MODE_IRQ = 0x12,
   ARM_CPU_MODE_SVC = 0x13,
+  ARM_CPU_MODE_MON = 0x16,
   ARM_CPU_MODE_ABT = 0x17,
+  ARM_CPU_MODE_HYP = 0x1a,
   ARM_CPU_MODE_UND = 0x1b,
   ARM_CPU_MODE_SYS = 0x1f
 };
@@ -631,6 +633,8 @@
     ARM_FEATURE_CBAR, /* has cp15 CBAR */
     ARM_FEATURE_CRC, /* ARMv8 CRC instructions */
     ARM_FEATURE_CBAR_RO, /* has cp15 CBAR and it is read-only */
+    ARM_FEATURE_EL2, /* has EL2 Virtualization support */
+    ARM_FEATURE_EL3, /* has EL3 Secure monitor support */
 };
 
 static inline int arm_feature(CPUARMState *env, int feature)
@@ -1080,12 +1084,12 @@
 #define cpu_list arm_cpu_list
 
 /* MMU modes definitions */
-#define MMU_MODE0_SUFFIX _kernel
-#define MMU_MODE1_SUFFIX _user
-#define MMU_USER_IDX 1
+#define MMU_MODE0_SUFFIX _user
+#define MMU_MODE1_SUFFIX _kernel
+#define MMU_USER_IDX 0
 static inline int cpu_mmu_index (CPUARMState *env)
 {
-    return arm_current_pl(env) ? 0 : 1;
+    return arm_current_pl(env);
 }
 
 #include "exec/cpu-all.h"
@@ -1195,26 +1199,4 @@
     }
 }
 
-/* Load an instruction and return it in the standard little-endian order */
-static inline uint32_t arm_ldl_code(CPUARMState *env, target_ulong addr,
-                                    bool do_swap)
-{
-    uint32_t insn = cpu_ldl_code(env, addr);
-    if (do_swap) {
-        return bswap32(insn);
-    }
-    return insn;
-}
-
-/* Ditto, for a halfword (Thumb) instruction */
-static inline uint16_t arm_lduw_code(CPUARMState *env, target_ulong addr,
-                                     bool do_swap)
-{
-    uint16_t insn = cpu_lduw_code(env, addr);
-    if (do_swap) {
-        return bswap16(insn);
-    }
-    return insn;
-}
-
 #endif
diff --git a/target-arm/crypto_helper.c b/target-arm/crypto_helper.c
index f94be69..d8898ed 100644
--- a/target-arm/crypto_helper.c
+++ b/target-arm/crypto_helper.c
@@ -13,7 +13,7 @@
 
 #include "cpu.h"
 #include "exec/exec-all.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 union AES_STATE {
     uint8_t    bytes[16];
diff --git a/target-arm/helper-a64.c b/target-arm/helper-a64.c
index bf921cc..cccda74 100644
--- a/target-arm/helper-a64.c
+++ b/target-arm/helper-a64.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "exec/gdbstub.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 #include "sysemu/sysemu.h"
 #include "qemu/bitops.h"
@@ -443,7 +443,7 @@
 {
     ARMCPU *cpu = ARM_CPU(cs);
     CPUARMState *env = &cpu->env;
-    target_ulong addr = env->cp15.c12_vbar;
+    target_ulong addr = env->cp15.vbar_el[1];
     int i;
 
     if (arm_current_pl(env) == 0) {
@@ -464,7 +464,7 @@
                       env->exception.syndrome);
     }
 
-    env->cp15.esr_el1 = env->exception.syndrome;
+    env->cp15.esr_el[1] = env->exception.syndrome;
     env->cp15.far_el1 = env->exception.vaddress;
 
     switch (cs->exception_index) {
@@ -488,16 +488,16 @@
     }
 
     if (is_a64(env)) {
-        env->banked_spsr[0] = pstate_read(env);
+        env->banked_spsr[aarch64_banked_spsr_index(1)] = pstate_read(env);
         env->sp_el[arm_current_pl(env)] = env->xregs[31];
         env->xregs[31] = env->sp_el[1];
-        env->elr_el1 = env->pc;
+        env->elr_el[1] = env->pc;
     } else {
         env->banked_spsr[0] = cpsr_read(env);
         if (!env->thumb) {
-            env->cp15.esr_el1 |= 1 << 25;
+            env->cp15.esr_el[1] |= 1 << 25;
         }
-        env->elr_el1 = env->regs[15];
+        env->elr_el[1] = env->regs[15];
 
         for (i = 0; i < 15; i++) {
             env->xregs[i] = env->regs[i];
diff --git a/target-arm/helper.c b/target-arm/helper.c
index 417161e..95af624 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -1,17 +1,17 @@
 #include "cpu.h"
 #include "internals.h"
 #include "exec/gdbstub.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 #include "sysemu/arch_init.h"
 #include "sysemu/sysemu.h"
 #include "qemu/bitops.h"
 #include "qemu/crc32c.h"
+#include "exec/cpu_ldst.h"
+#include "arm_ldst.h"
 #include <zlib.h> /* For crc32 */
 
 #ifndef CONFIG_USER_ONLY
-#include "exec/softmmu_exec.h"
-
 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
                                 int access_type, int is_user,
                                 hwaddr *phys_ptr, int *prot,
@@ -477,11 +477,35 @@
 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
-    if (env->cp15.c1_coproc != value) {
-        env->cp15.c1_coproc = value;
-        /* ??? Is this safe when called from within a TB?  */
-        tb_flush(env);
+    uint32_t mask = 0;
+
+    /* In ARMv8 most bits of CPACR_EL1 are RES0. */
+    if (!arm_feature(env, ARM_FEATURE_V8)) {
+        /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
+         * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
+         * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
+         */
+        if (arm_feature(env, ARM_FEATURE_VFP)) {
+            /* VFP coprocessor: cp10 & cp11 [23:20] */
+            mask |= (1 << 31) | (1 << 30) | (0xf << 20);
+
+            if (!arm_feature(env, ARM_FEATURE_NEON)) {
+                /* ASEDIS [31] bit is RAO/WI */
+                value |= (1 << 31);
+            }
+
+            /* VFPv3 and upwards with NEON implement 32 double precision
+             * registers (D0-D31).
+             */
+            if (!arm_feature(env, ARM_FEATURE_NEON) ||
+                    !arm_feature(env, ARM_FEATURE_VFP3)) {
+                /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
+                value |= (1 << 30);
+            }
+        }
+        value &= mask;
     }
+    env->cp15.c1_coproc = value;
 }
 
 static const ARMCPRegInfo v6_cp_reginfo[] = {
@@ -657,7 +681,7 @@
      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
      */
-    env->cp15.c12_vbar = value & ~0x1FULL;
+    raw_write(env, ri, value & ~0x1FULL);
 }
 
 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
@@ -766,7 +790,7 @@
     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .writefn = vbar_write,
-      .fieldoffset = offsetof(CPUARMState, cp15.c12_vbar),
+      .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[1]),
       .resetvalue = 0 },
     { .name = "SCR", .cp = 15, .crn = 1, .crm = 1, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_scr),
@@ -1452,7 +1476,7 @@
 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el1),
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
       .resetfn = arm_cp_reset_ignore, },
     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
       .access = PL1_RW,
@@ -1460,7 +1484,7 @@
     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW,
-      .fieldoffset = offsetof(CPUARMState, cp15.esr_el1), .resetvalue = 0, },
+      .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
@@ -1521,7 +1545,7 @@
 static const ARMCPRegInfo omap_cp_reginfo[] = {
     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
-      .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el1),
+      .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
       .resetvalue = 0, },
     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
       .access = PL1_RW, .type = ARM_CP_NOP },
@@ -2055,7 +2079,8 @@
     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
-      .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, elr_el1) },
+      .access = PL1_RW,
+      .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
       .type = ARM_CP_NO_MIGRATE,
       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
@@ -2076,6 +2101,51 @@
     REGINFO_SENTINEL
 };
 
+/* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
+static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
+    { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
+      .access = PL2_RW,
+      .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
+    REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
+    { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
+      .access = PL2_RW,
+      .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
+    { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
+    { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
+      .access = PL2_RW, .writefn = vbar_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
+      .resetvalue = 0 },
+    REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo v8_el3_cp_reginfo[] = {
+    { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
+      .access = PL3_RW,
+      .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
+    { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
+      .type = ARM_CP_NO_MIGRATE,
+      .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
+      .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
+    { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
+      .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
+      .access = PL3_RW, .writefn = vbar_write,
+      .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
+      .resetvalue = 0 },
+    REGINFO_SENTINEL
+};
+
 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
                         uint64_t value)
 {
@@ -2327,6 +2397,19 @@
         define_arm_cp_regs(cpu, v8_cp_reginfo);
         define_aarch64_debug_regs(cpu);
     }
+    if (arm_feature(env, ARM_FEATURE_EL2)) {
+        define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
+    } else {
+        /* If EL2 is missing but higher ELs are enabled, we need to
+         * register the no_el2 reginfos.
+         */
+        if (arm_feature(env, ARM_FEATURE_EL3)) {
+            define_arm_cp_regs(cpu, v8_el3_no_el2_cp_reginfo);
+        }
+    }
+    if (arm_feature(env, ARM_FEATURE_EL3)) {
+        define_arm_cp_regs(cpu, v8_el3_cp_reginfo);
+    }
     if (arm_feature(env, ARM_FEATURE_MPU)) {
         /* These are the MPU registers prior to PMSAv6. Any new
          * PMSA core later than the ARM946 will require that we
@@ -3083,6 +3166,10 @@
         return 4;
     case ARM_CPU_MODE_FIQ:
         return 5;
+    case ARM_CPU_MODE_HYP:
+        return 6;
+    case ARM_CPU_MODE_MON:
+        return 7;
     }
     hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
 }
@@ -3337,11 +3424,11 @@
         offset = 4;
         break;
     case EXCP_DATA_ABORT:
-        env->cp15.esr_el1 = env->exception.fsr;
+        env->cp15.esr_el[1] = env->exception.fsr;
         env->cp15.far_el1 = deposit64(env->cp15.far_el1, 0, 32,
                                       env->exception.vaddress);
         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
-                      (uint32_t)env->cp15.esr_el1,
+                      (uint32_t)env->cp15.esr_el[1],
                       (uint32_t)env->exception.vaddress);
         new_mode = ARM_CPU_MODE_ABT;
         addr = 0x10;
@@ -3378,7 +3465,7 @@
          * and is never in monitor mode this feature is always active.
          * Note: only bits 31:5 are valid.
          */
-        addr += env->cp15.c12_vbar;
+        addr += env->cp15.vbar_el[1];
     }
     switch_mode (env, new_mode);
     env->spsr = cpsr_read(env);
diff --git a/target-arm/helper.h b/target-arm/helper.h
index a5449e7..b63fd0f 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_FLAGS_1(sxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_FLAGS_1(uxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
@@ -521,5 +519,3 @@
 #ifdef TARGET_AARCH64
 #include "helper-a64.h"
 #endif
-
-#include "exec/def-helper.h"
diff --git a/target-arm/internals.h b/target-arm/internals.h
index d63a975..564b5fa 100644
--- a/target-arm/internals.h
+++ b/target-arm/internals.h
@@ -75,6 +75,20 @@
  */
 #define GTIMER_SCALE 16
 
+/*
+ * For AArch64, map a given EL to an index in the banked_spsr array.
+ */
+static inline unsigned int aarch64_banked_spsr_index(unsigned int el)
+{
+    static const unsigned int map[4] = {
+        [1] = 0, /* EL1.  */
+        [2] = 6, /* EL2.  */
+        [3] = 7, /* EL3.  */
+    };
+    assert(el >= 1 && el <= 3);
+    return map[el];
+}
+
 int bank_number(int mode);
 void switch_mode(CPUARMState *, int);
 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu);
@@ -93,6 +107,7 @@
 
 static inline void update_spsel(CPUARMState *env, uint32_t imm)
 {
+    unsigned int cur_el = arm_current_pl(env);
     /* Update PSTATE SPSel bit; this requires us to update the
      * working stack pointer in xregs[31].
      */
@@ -101,17 +116,17 @@
     }
     env->pstate = deposit32(env->pstate, 0, 1, imm);
 
-    /* EL0 has no access rights to update SPSel, and this code
-     * assumes we are updating SP for EL1 while running as EL1.
+    /* We rely on illegal updates to SPsel from EL0 to get trapped
+     * at translation time.
      */
-    assert(arm_current_pl(env) == 1);
+    assert(cur_el >= 1 && cur_el <= 3);
     if (env->pstate & PSTATE_SP) {
         /* Switch from using SP_EL0 to using SP_ELx */
         env->sp_el[0] = env->xregs[31];
-        env->xregs[31] = env->sp_el[1];
+        env->xregs[31] = env->sp_el[cur_el];
     } else {
         /* Switch from SP_EL0 to SP_ELx */
-        env->sp_el[1] = env->xregs[31];
+        env->sp_el[cur_el] = env->xregs[31];
         env->xregs[31] = env->sp_el[0];
     }
 }
diff --git a/target-arm/iwmmxt_helper.c b/target-arm/iwmmxt_helper.c
index e6cfa62..398cbcb 100644
--- a/target-arm/iwmmxt_helper.c
+++ b/target-arm/iwmmxt_helper.c
@@ -24,7 +24,7 @@
 
 #include "cpu.h"
 #include "exec/exec-all.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /* iwMMXt macros extracted from GNU gdb.  */
 
diff --git a/target-arm/kvm64.c b/target-arm/kvm64.c
index c729b9e..70f311b 100644
--- a/target-arm/kvm64.c
+++ b/target-arm/kvm64.c
@@ -161,7 +161,7 @@
     }
 
     reg.id = AARCH64_CORE_REG(elr_el1);
-    reg.addr = (uintptr_t) &env->elr_el1;
+    reg.addr = (uintptr_t) &env->elr_el[1];
     ret = kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, &reg);
     if (ret) {
         return ret;
@@ -241,7 +241,7 @@
     }
 
     reg.id = AARCH64_CORE_REG(elr_el1);
-    reg.addr = (uintptr_t) &env->elr_el1;
+    reg.addr = (uintptr_t) &env->elr_el[1];
     ret = kvm_vcpu_ioctl(cs, KVM_GET_ONE_REG, &reg);
     if (ret) {
         return ret;
diff --git a/target-arm/machine.c b/target-arm/machine.c
index 5092dcd..3bcc7cc 100644
--- a/target-arm/machine.c
+++ b/target-arm/machine.c
@@ -218,8 +218,8 @@
 
 const VMStateDescription vmstate_arm_cpu = {
     .name = "cpu",
-    .version_id = 17,
-    .minimum_version_id = 17,
+    .version_id = 20,
+    .minimum_version_id = 20,
     .pre_save = cpu_pre_save,
     .post_load = cpu_post_load,
     .fields = (VMStateField[]) {
@@ -233,13 +233,13 @@
             .offset = 0,
         },
         VMSTATE_UINT32(env.spsr, ARMCPU),
-        VMSTATE_UINT64_ARRAY(env.banked_spsr, ARMCPU, 6),
+        VMSTATE_UINT64_ARRAY(env.banked_spsr, ARMCPU, 8),
         VMSTATE_UINT32_ARRAY(env.banked_r13, ARMCPU, 6),
         VMSTATE_UINT32_ARRAY(env.banked_r14, ARMCPU, 6),
         VMSTATE_UINT32_ARRAY(env.usr_regs, ARMCPU, 5),
         VMSTATE_UINT32_ARRAY(env.fiq_regs, ARMCPU, 5),
-        VMSTATE_UINT64(env.elr_el1, ARMCPU),
-        VMSTATE_UINT64_ARRAY(env.sp_el, ARMCPU, 2),
+        VMSTATE_UINT64_ARRAY(env.elr_el, ARMCPU, 4),
+        VMSTATE_UINT64_ARRAY(env.sp_el, ARMCPU, 4),
         /* The length-check must come before the arrays to avoid
          * incoming data possibly overflowing the array.
          */
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index 8d6f9a9..492e500 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -11,7 +11,7 @@
 
 #include "cpu.h"
 #include "exec/exec-all.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index fb90676..9c1ef52 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -17,8 +17,9 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "internals.h"
+#include "exec/cpu_ldst.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
@@ -56,22 +57,6 @@
 
 #if !defined(CONFIG_USER_ONLY)
 
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 /* try to fill the TLB and return an exception if error. If retaddr is
  * NULL, it means that the function was called in C code (i.e. not
  * from generated code or from helper.c)
@@ -386,11 +371,13 @@
 
 void HELPER(exception_return)(CPUARMState *env)
 {
-    uint32_t spsr = env->banked_spsr[0];
+    int cur_el = arm_current_pl(env);
+    unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
+    uint32_t spsr = env->banked_spsr[spsr_idx];
     int new_el, i;
 
     if (env->pstate & PSTATE_SP) {
-        env->sp_el[1] = env->xregs[31];
+        env->sp_el[cur_el] = env->xregs[31];
     } else {
         env->sp_el[0] = env->xregs[31];
     }
@@ -398,6 +385,7 @@
     env->exclusive_addr = -1;
 
     if (spsr & PSTATE_nRW) {
+        /* TODO: We currently assume EL1/2/3 are running in AArch64.  */
         env->aarch64 = 0;
         new_el = 0;
         env->uncached_cpsr = 0x10;
@@ -406,11 +394,14 @@
             env->regs[i] = env->xregs[i];
         }
 
-        env->regs[15] = env->elr_el1 & ~0x1;
+        env->regs[15] = env->elr_el[1] & ~0x1;
     } else {
         new_el = extract32(spsr, 2, 2);
-        if (new_el > 1) {
-            /* Return to unimplemented EL */
+        if (new_el > cur_el
+            || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
+            /* Disallow return to an EL which is unimplemented or higher
+             * than the current one.
+             */
             goto illegal_return;
         }
         if (extract32(spsr, 1, 1)) {
@@ -424,7 +415,7 @@
         env->aarch64 = 1;
         pstate_write(env, spsr);
         env->xregs[31] = env->sp_el[new_el];
-        env->pc = env->elr_el1;
+        env->pc = env->elr_el[cur_el];
     }
 
     return;
@@ -438,7 +429,7 @@
      * no change to exception level, execution state or stack pointer
      */
     env->pstate |= PSTATE_IL;
-    env->pc = env->elr_el1;
+    env->pc = env->elr_el[cur_el];
     spsr &= PSTATE_NZCV | PSTATE_DAIF;
     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
     pstate_write(env, spsr);
diff --git a/target-arm/translate-a64.c b/target-arm/translate-a64.c
index b62db4d..a9c4633 100644
--- a/target-arm/translate-a64.c
+++ b/target-arm/translate-a64.c
@@ -25,15 +25,15 @@
 #include "cpu.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
+#include "arm_ldst.h"
 #include "translate.h"
 #include "internals.h"
 #include "qemu/host-utils.h"
 
 #include "exec/gen-icount.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 static TCGv_i64 cpu_X[32];
 static TCGv_i64 cpu_pc;
@@ -162,15 +162,6 @@
     }
 }
 
-static int get_mem_index(DisasContext *s)
-{
-#ifdef CONFIG_USER_ONLY
-    return 1;
-#else
-    return s->user;
-#endif
-}
-
 void gen_a64_set_pc_im(uint64_t val)
 {
     tcg_gen_movi_i64(cpu_pc, val);
@@ -1516,6 +1507,10 @@
         tcg_gen_movi_i64(cpu_reg(s, 30), s->pc);
         break;
     case 4: /* ERET */
+        if (s->current_pl == 0) {
+            unallocated_encoding(s);
+            return;
+        }
         gen_helper_exception_return(cpu_env);
         s->is_jmp = DISAS_JUMP;
         return;
diff --git a/target-arm/translate.c b/target-arm/translate.c
index a4d920b..d499caa 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -30,10 +30,10 @@
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "qemu/bitops.h"
+#include "arm_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #define ENABLE_ARCH_4T    arm_feature(env, ARM_FEATURE_V4T)
 #define ENABLE_ARCH_5     arm_feature(env, ARM_FEATURE_V5)
@@ -1165,18 +1165,18 @@
 static inline void gen_vfp_ld(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_ld64(cpu_F0d, addr, IS_USER(s));
+        gen_aa32_ld64(cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_ld32u(cpu_F0s, addr, IS_USER(s));
+        gen_aa32_ld32u(cpu_F0s, addr, get_mem_index(s));
     }
 }
 
 static inline void gen_vfp_st(DisasContext *s, int dp, TCGv_i32 addr)
 {
     if (dp) {
-        gen_aa32_st64(cpu_F0d, addr, IS_USER(s));
+        gen_aa32_st64(cpu_F0d, addr, get_mem_index(s));
     } else {
-        gen_aa32_st32(cpu_F0s, addr, IS_USER(s));
+        gen_aa32_st32(cpu_F0s, addr, get_mem_index(s));
     }
 }
 
@@ -1514,24 +1514,24 @@
         if (insn & ARM_CP_RW_BIT) {
             if ((insn >> 28) == 0xf) {			/* WLDRW wCx */
                 tmp = tcg_temp_new_i32();
-                gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                 iwmmxt_store_creg(wrd, tmp);
             } else {
                 i = 1;
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WLDRD */
-                        gen_aa32_ld64(cpu_M0, addr, IS_USER(s));
+                        gen_aa32_ld64(cpu_M0, addr, get_mem_index(s));
                         i = 0;
                     } else {				/* WLDRW wRd */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     }
                 } else {
                     tmp = tcg_temp_new_i32();
                     if (insn & (1 << 22)) {		/* WLDRH */
-                        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                     } else {				/* WLDRB */
-                        gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                     }
                 }
                 if (i) {
@@ -1543,24 +1543,24 @@
         } else {
             if ((insn >> 28) == 0xf) {			/* WSTRW wCx */
                 tmp = iwmmxt_load_creg(wrd);
-                gen_aa32_st32(tmp, addr, IS_USER(s));
+                gen_aa32_st32(tmp, addr, get_mem_index(s));
             } else {
                 gen_op_iwmmxt_movq_M0_wRn(wrd);
                 tmp = tcg_temp_new_i32();
                 if (insn & (1 << 8)) {
                     if (insn & (1 << 22)) {		/* WSTRD */
-                        gen_aa32_st64(cpu_M0, addr, IS_USER(s));
+                        gen_aa32_st64(cpu_M0, addr, get_mem_index(s));
                     } else {				/* WSTRW wRd */
                         tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                     }
                 } else {
                     if (insn & (1 << 22)) {		/* WSTRH */
                         tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st16(tmp, addr, IS_USER(s));
+                        gen_aa32_st16(tmp, addr, get_mem_index(s));
                     } else {				/* WSTRB */
                         tcg_gen_trunc_i64_i32(tmp, cpu_M0);
-                        gen_aa32_st8(tmp, addr, IS_USER(s));
+                        gen_aa32_st8(tmp, addr, get_mem_index(s));
                     }
                 }
             }
@@ -2625,15 +2625,15 @@
     TCGv_i32 tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, IS_USER(s));
+        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
         gen_neon_dup_u8(tmp, 0);
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
         gen_neon_dup_low16(tmp);
         break;
     case 2:
-        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
         break;
     default: /* Avoid compiler warnings.  */
         abort();
@@ -4304,11 +4304,11 @@
             if (size == 3) {
                 tmp64 = tcg_temp_new_i64();
                 if (load) {
-                    gen_aa32_ld64(tmp64, addr, IS_USER(s));
+                    gen_aa32_ld64(tmp64, addr, get_mem_index(s));
                     neon_store_reg64(tmp64, rd);
                 } else {
                     neon_load_reg64(tmp64, rd);
-                    gen_aa32_st64(tmp64, addr, IS_USER(s));
+                    gen_aa32_st64(tmp64, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i64(tmp64);
                 tcg_gen_addi_i32(addr, addr, stride);
@@ -4317,21 +4317,21 @@
                     if (size == 2) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                             neon_store_reg(rd, pass, tmp);
                         } else {
                             tmp = neon_load_reg(rd, pass);
-                            gen_aa32_st32(tmp, addr, IS_USER(s));
+                            gen_aa32_st32(tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         tcg_gen_addi_i32(addr, addr, stride);
                     } else if (size == 1) {
                         if (load) {
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tmp2 = tcg_temp_new_i32();
-                            gen_aa32_ld16u(tmp2, addr, IS_USER(s));
+                            gen_aa32_ld16u(tmp2, addr, get_mem_index(s));
                             tcg_gen_addi_i32(addr, addr, stride);
                             tcg_gen_shli_i32(tmp2, tmp2, 16);
                             tcg_gen_or_i32(tmp, tmp, tmp2);
@@ -4341,10 +4341,10 @@
                             tmp = neon_load_reg(rd, pass);
                             tmp2 = tcg_temp_new_i32();
                             tcg_gen_shri_i32(tmp2, tmp, 16);
-                            gen_aa32_st16(tmp, addr, IS_USER(s));
+                            gen_aa32_st16(tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                             tcg_gen_addi_i32(addr, addr, stride);
-                            gen_aa32_st16(tmp2, addr, IS_USER(s));
+                            gen_aa32_st16(tmp2, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp2);
                             tcg_gen_addi_i32(addr, addr, stride);
                         }
@@ -4353,7 +4353,7 @@
                             TCGV_UNUSED_I32(tmp2);
                             for (n = 0; n < 4; n++) {
                                 tmp = tcg_temp_new_i32();
-                                gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                                gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                                 tcg_gen_addi_i32(addr, addr, stride);
                                 if (n == 0) {
                                     tmp2 = tmp;
@@ -4373,7 +4373,7 @@
                                 } else {
                                     tcg_gen_shri_i32(tmp, tmp2, n * 8);
                                 }
-                                gen_aa32_st8(tmp, addr, IS_USER(s));
+                                gen_aa32_st8(tmp, addr, get_mem_index(s));
                                 tcg_temp_free_i32(tmp);
                                 tcg_gen_addi_i32(addr, addr, stride);
                             }
@@ -4497,13 +4497,13 @@
                     tmp = tcg_temp_new_i32();
                     switch (size) {
                     case 0:
-                        gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                         break;
                     default: /* Avoid compiler warnings.  */
                         abort();
@@ -4521,13 +4521,13 @@
                         tcg_gen_shri_i32(tmp, tmp, shift);
                     switch (size) {
                     case 0:
-                        gen_aa32_st8(tmp, addr, IS_USER(s));
+                        gen_aa32_st8(tmp, addr, get_mem_index(s));
                         break;
                     case 1:
-                        gen_aa32_st16(tmp, addr, IS_USER(s));
+                        gen_aa32_st16(tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                         break;
                     }
                     tcg_temp_free_i32(tmp);
@@ -7173,14 +7173,14 @@
 
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, IS_USER(s));
+        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7191,7 +7191,7 @@
         TCGv_i32 tmp3 = tcg_temp_new_i32();
 
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, IS_USER(s));
+        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(cpu_exclusive_val, tmp, tmp3);
         store_reg(s, rt2, tmp3);
@@ -7242,14 +7242,14 @@
     tmp = tcg_temp_new_i32();
     switch (size) {
     case 0:
-        gen_aa32_ld8u(tmp, addr, IS_USER(s));
+        gen_aa32_ld8u(tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7260,7 +7260,7 @@
         TCGv_i32 tmp2 = tcg_temp_new_i32();
         TCGv_i32 tmp3 = tcg_temp_new_i32();
         tcg_gen_addi_i32(tmp2, addr, 4);
-        gen_aa32_ld32u(tmp3, tmp2, IS_USER(s));
+        gen_aa32_ld32u(tmp3, tmp2, get_mem_index(s));
         tcg_temp_free_i32(tmp2);
         tcg_gen_concat_i32_i64(val64, tmp, tmp3);
         tcg_temp_free_i32(tmp3);
@@ -7275,14 +7275,14 @@
     tmp = load_reg(s, rt);
     switch (size) {
     case 0:
-        gen_aa32_st8(tmp, addr, IS_USER(s));
+        gen_aa32_st8(tmp, addr, get_mem_index(s));
         break;
     case 1:
-        gen_aa32_st16(tmp, addr, IS_USER(s));
+        gen_aa32_st16(tmp, addr, get_mem_index(s));
         break;
     case 2:
     case 3:
-        gen_aa32_st32(tmp, addr, IS_USER(s));
+        gen_aa32_st32(tmp, addr, get_mem_index(s));
         break;
     default:
         abort();
@@ -7291,7 +7291,7 @@
     if (size == 3) {
         tcg_gen_addi_i32(addr, addr, 4);
         tmp = load_reg(s, rt2);
-        gen_aa32_st32(tmp, addr, IS_USER(s));
+        gen_aa32_st32(tmp, addr, get_mem_index(s));
         tcg_temp_free_i32(tmp);
     }
     tcg_gen_movi_i32(cpu_R[rd], 0);
@@ -7338,11 +7338,11 @@
     }
     tcg_gen_addi_i32(addr, addr, offset);
     tmp = load_reg(s, 14);
-    gen_aa32_st32(tmp, addr, 0);
+    gen_aa32_st32(tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     tmp = load_cpu_field(spsr);
     tcg_gen_addi_i32(addr, addr, 4);
-    gen_aa32_st32(tmp, addr, 0);
+    gen_aa32_st32(tmp, addr, get_mem_index(s));
     tcg_temp_free_i32(tmp);
     if (writeback) {
         switch (amode) {
@@ -7495,10 +7495,10 @@
                 tcg_gen_addi_i32(addr, addr, offset);
             /* Load PC into tmp and CPSR into tmp2.  */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, 0);
+            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
             tcg_gen_addi_i32(addr, addr, 4);
             tmp2 = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp2, addr, 0);
+            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
             if (insn & (1 << 21)) {
                 /* Base writeback.  */
                 switch (i) {
@@ -8087,13 +8087,13 @@
                                 tmp = tcg_temp_new_i32();
                                 switch (op1) {
                                 case 0: /* lda */
-                                    gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                                     break;
                                 case 2: /* ldab */
-                                    gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                                     break;
                                 case 3: /* ldah */
-                                    gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8104,13 +8104,13 @@
                                 tmp = load_reg(s, rm);
                                 switch (op1) {
                                 case 0: /* stl */
-                                    gen_aa32_st32(tmp, addr, IS_USER(s));
+                                    gen_aa32_st32(tmp, addr, get_mem_index(s));
                                     break;
                                 case 2: /* stlb */
-                                    gen_aa32_st8(tmp, addr, IS_USER(s));
+                                    gen_aa32_st8(tmp, addr, get_mem_index(s));
                                     break;
                                 case 3: /* stlh */
-                                    gen_aa32_st16(tmp, addr, IS_USER(s));
+                                    gen_aa32_st16(tmp, addr, get_mem_index(s));
                                     break;
                                 default:
                                     abort();
@@ -8165,11 +8165,11 @@
                         tmp = load_reg(s, rm);
                         tmp2 = tcg_temp_new_i32();
                         if (insn & (1 << 22)) {
-                            gen_aa32_ld8u(tmp2, addr, IS_USER(s));
-                            gen_aa32_st8(tmp, addr, IS_USER(s));
+                            gen_aa32_ld8u(tmp2, addr, get_mem_index(s));
+                            gen_aa32_st8(tmp, addr, get_mem_index(s));
                         } else {
-                            gen_aa32_ld32u(tmp2, addr, IS_USER(s));
-                            gen_aa32_st32(tmp, addr, IS_USER(s));
+                            gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
+                            gen_aa32_st32(tmp, addr, get_mem_index(s));
                         }
                         tcg_temp_free_i32(tmp);
                         tcg_temp_free_i32(addr);
@@ -8191,14 +8191,14 @@
                     tmp = tcg_temp_new_i32();
                     switch(sh) {
                     case 1:
-                        gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                         break;
                     case 2:
-                        gen_aa32_ld8s(tmp, addr, IS_USER(s));
+                        gen_aa32_ld8s(tmp, addr, get_mem_index(s));
                         break;
                     default:
                     case 3:
-                        gen_aa32_ld16s(tmp, addr, IS_USER(s));
+                        gen_aa32_ld16s(tmp, addr, get_mem_index(s));
                         break;
                     }
                     load = 1;
@@ -8208,21 +8208,21 @@
                     if (sh & 1) {
                         /* store */
                         tmp = load_reg(s, rd);
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = load_reg(s, rd + 1);
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                         load = 0;
                     } else {
                         /* load */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                         store_reg(s, rd, tmp);
                         tcg_gen_addi_i32(addr, addr, 4);
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                         rd++;
                         load = 1;
                     }
@@ -8230,7 +8230,7 @@
                 } else {
                     /* store */
                     tmp = load_reg(s, rd);
-                    gen_aa32_st16(tmp, addr, IS_USER(s));
+                    gen_aa32_st16(tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                     load = 0;
                 }
@@ -8568,7 +8568,12 @@
             rn = (insn >> 16) & 0xf;
             rd = (insn >> 12) & 0xf;
             tmp2 = load_reg(s, rn);
-            i = (IS_USER(s) || (insn & 0x01200000) == 0x00200000);
+            if ((insn & 0x01200000) == 0x00200000) {
+                /* ldrt/strt */
+                i = MMU_USER_IDX;
+            } else {
+                i = get_mem_index(s);
+            }
             if (insn & (1 << 24))
                 gen_add_data_offset(s, insn, tmp2);
             if (insn & (1 << 20)) {
@@ -8652,7 +8657,7 @@
                         if (insn & (1 << 20)) {
                             /* load */
                             tmp = tcg_temp_new_i32();
-                            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                             if (user) {
                                 tmp2 = tcg_const_i32(i);
                                 gen_helper_set_user_reg(cpu_env, tmp2, tmp);
@@ -8679,7 +8684,7 @@
                             } else {
                                 tmp = load_reg(s, i);
                             }
-                            gen_aa32_st32(tmp, addr, IS_USER(s));
+                            gen_aa32_st32(tmp, addr, get_mem_index(s));
                             tcg_temp_free_i32(tmp);
                         }
                         j++;
@@ -8945,20 +8950,20 @@
                 if (insn & (1 << 20)) {
                     /* ldrd */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     store_reg(s, rs, tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     store_reg(s, rd, tmp);
                 } else {
                     /* strd */
                     tmp = load_reg(s, rs);
-                    gen_aa32_st32(tmp, addr, IS_USER(s));
+                    gen_aa32_st32(tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = load_reg(s, rd);
-                    gen_aa32_st32(tmp, addr, IS_USER(s));
+                    gen_aa32_st32(tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 if (insn & (1 << 21)) {
@@ -8996,11 +9001,11 @@
                     tcg_gen_add_i32(addr, addr, tmp);
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                 } else { /* tbb */
                     tcg_temp_free_i32(tmp);
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                 }
                 tcg_temp_free_i32(addr);
                 tcg_gen_shli_i32(tmp, tmp, 1);
@@ -9037,13 +9042,13 @@
                         tmp = tcg_temp_new_i32();
                         switch (op) {
                         case 0: /* ldab */
-                            gen_aa32_ld8u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* ldah */
-                            gen_aa32_ld16u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* lda */
-                            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9053,13 +9058,13 @@
                         tmp = load_reg(s, rs);
                         switch (op) {
                         case 0: /* stlb */
-                            gen_aa32_st8(tmp, addr, IS_USER(s));
+                            gen_aa32_st8(tmp, addr, get_mem_index(s));
                             break;
                         case 1: /* stlh */
-                            gen_aa32_st16(tmp, addr, IS_USER(s));
+                            gen_aa32_st16(tmp, addr, get_mem_index(s));
                             break;
                         case 2: /* stl */
-                            gen_aa32_st32(tmp, addr, IS_USER(s));
+                            gen_aa32_st32(tmp, addr, get_mem_index(s));
                             break;
                         default:
                             abort();
@@ -9087,10 +9092,10 @@
                         tcg_gen_addi_i32(addr, addr, -8);
                     /* Load PC into tmp and CPSR into tmp2.  */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, 0);
+                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp2 = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp2, addr, 0);
+                    gen_aa32_ld32u(tmp2, addr, get_mem_index(s));
                     if (insn & (1 << 21)) {
                         /* Base writeback.  */
                         if (insn & (1 << 24)) {
@@ -9129,7 +9134,7 @@
                     if (insn & (1 << 20)) {
                         /* Load.  */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                         if (i == 15) {
                             gen_bx(s, tmp);
                         } else if (i == rn) {
@@ -9141,7 +9146,7 @@
                     } else {
                         /* Store.  */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     tcg_gen_addi_i32(addr, addr, 4);
@@ -9841,7 +9846,7 @@
         {
         int postinc = 0;
         int writeback = 0;
-        int user;
+        int memidx;
         if ((insn & 0x01100000) == 0x01000000) {
             if (disas_neon_ls_insn(env, s, insn))
                 goto illegal_op;
@@ -9885,7 +9890,7 @@
                 return 1;
             }
         }
-        user = IS_USER(s);
+        memidx = get_mem_index(s);
         if (rn == 15) {
             addr = tcg_temp_new_i32();
             /* PC relative.  */
@@ -9922,7 +9927,7 @@
                     break;
                 case 0xe: /* User privilege.  */
                     tcg_gen_addi_i32(addr, addr, imm);
-                    user = 1;
+                    memidx = MMU_USER_IDX;
                     break;
                 case 0x9: /* Post-decrement.  */
                     imm = -imm;
@@ -9949,19 +9954,19 @@
             tmp = tcg_temp_new_i32();
             switch (op) {
             case 0:
-                gen_aa32_ld8u(tmp, addr, user);
+                gen_aa32_ld8u(tmp, addr, memidx);
                 break;
             case 4:
-                gen_aa32_ld8s(tmp, addr, user);
+                gen_aa32_ld8s(tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_ld16u(tmp, addr, user);
+                gen_aa32_ld16u(tmp, addr, memidx);
                 break;
             case 5:
-                gen_aa32_ld16s(tmp, addr, user);
+                gen_aa32_ld16s(tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_ld32u(tmp, addr, user);
+                gen_aa32_ld32u(tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -9978,13 +9983,13 @@
             tmp = load_reg(s, rs);
             switch (op) {
             case 0:
-                gen_aa32_st8(tmp, addr, user);
+                gen_aa32_st8(tmp, addr, memidx);
                 break;
             case 1:
-                gen_aa32_st16(tmp, addr, user);
+                gen_aa32_st16(tmp, addr, memidx);
                 break;
             case 2:
-                gen_aa32_st32(tmp, addr, user);
+                gen_aa32_st32(tmp, addr, memidx);
                 break;
             default:
                 tcg_temp_free_i32(tmp);
@@ -10121,7 +10126,7 @@
             addr = tcg_temp_new_i32();
             tcg_gen_movi_i32(addr, val);
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(addr);
             store_reg(s, rd, tmp);
             break;
@@ -10324,28 +10329,28 @@
 
         switch (op) {
         case 0: /* str */
-            gen_aa32_st32(tmp, addr, IS_USER(s));
+            gen_aa32_st32(tmp, addr, get_mem_index(s));
             break;
         case 1: /* strh */
-            gen_aa32_st16(tmp, addr, IS_USER(s));
+            gen_aa32_st16(tmp, addr, get_mem_index(s));
             break;
         case 2: /* strb */
-            gen_aa32_st8(tmp, addr, IS_USER(s));
+            gen_aa32_st8(tmp, addr, get_mem_index(s));
             break;
         case 3: /* ldrsb */
-            gen_aa32_ld8s(tmp, addr, IS_USER(s));
+            gen_aa32_ld8s(tmp, addr, get_mem_index(s));
             break;
         case 4: /* ldr */
-            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
             break;
         case 5: /* ldrh */
-            gen_aa32_ld16u(tmp, addr, IS_USER(s));
+            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
             break;
         case 6: /* ldrb */
-            gen_aa32_ld8u(tmp, addr, IS_USER(s));
+            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
             break;
         case 7: /* ldrsh */
-            gen_aa32_ld16s(tmp, addr, IS_USER(s));
+            gen_aa32_ld16s(tmp, addr, get_mem_index(s));
             break;
         }
         if (op >= 3) { /* load */
@@ -10367,12 +10372,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, IS_USER(s));
+            gen_aa32_st32(tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10389,12 +10394,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld8u(tmp, addr, IS_USER(s));
+            gen_aa32_ld8u(tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st8(tmp, addr, IS_USER(s));
+            gen_aa32_st8(tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10411,12 +10416,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld16u(tmp, addr, IS_USER(s));
+            gen_aa32_ld16u(tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st16(tmp, addr, IS_USER(s));
+            gen_aa32_st16(tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10432,12 +10437,12 @@
         if (insn & (1 << 11)) {
             /* load */
             tmp = tcg_temp_new_i32();
-            gen_aa32_ld32u(tmp, addr, IS_USER(s));
+            gen_aa32_ld32u(tmp, addr, get_mem_index(s));
             store_reg(s, rd, tmp);
         } else {
             /* store */
             tmp = load_reg(s, rd);
-            gen_aa32_st32(tmp, addr, IS_USER(s));
+            gen_aa32_st32(tmp, addr, get_mem_index(s));
             tcg_temp_free_i32(tmp);
         }
         tcg_temp_free_i32(addr);
@@ -10505,12 +10510,12 @@
                     if (insn & (1 << 11)) {
                         /* pop */
                         tmp = tcg_temp_new_i32();
-                        gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                        gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                         store_reg(s, i, tmp);
                     } else {
                         /* push */
                         tmp = load_reg(s, i);
-                        gen_aa32_st32(tmp, addr, IS_USER(s));
+                        gen_aa32_st32(tmp, addr, get_mem_index(s));
                         tcg_temp_free_i32(tmp);
                     }
                     /* advance to the next address.  */
@@ -10522,13 +10527,13 @@
                 if (insn & (1 << 11)) {
                     /* pop pc */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     /* don't set the pc until the rest of the instruction
                        has completed */
                 } else {
                     /* push lr */
                     tmp = load_reg(s, 14);
-                    gen_aa32_st32(tmp, addr, IS_USER(s));
+                    gen_aa32_st32(tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 tcg_gen_addi_i32(addr, addr, 4);
@@ -10657,7 +10662,7 @@
                 if (insn & (1 << 11)) {
                     /* load */
                     tmp = tcg_temp_new_i32();
-                    gen_aa32_ld32u(tmp, addr, IS_USER(s));
+                    gen_aa32_ld32u(tmp, addr, get_mem_index(s));
                     if (i == rn) {
                         loaded_var = tmp;
                     } else {
@@ -10666,7 +10671,7 @@
                 } else {
                     /* store */
                     tmp = load_reg(s, i);
-                    gen_aa32_st32(tmp, addr, IS_USER(s));
+                    gen_aa32_st32(tmp, addr, get_mem_index(s));
                     tcg_temp_free_i32(tmp);
                 }
                 /* advance to the next address */
@@ -11047,8 +11052,8 @@
 }
 
 static const char *cpu_mode_names[16] = {
-  "usr", "fiq", "irq", "svc", "???", "???", "???", "abt",
-  "???", "???", "???", "und", "???", "???", "???", "sys"
+  "usr", "fiq", "irq", "svc", "???", "???", "mon", "abt",
+  "???", "???", "hyp", "und", "???", "???", "???", "sys"
 };
 
 void arm_cpu_dump_state(CPUState *cs, FILE *f, fprintf_function cpu_fprintf,
diff --git a/target-arm/translate.h b/target-arm/translate.h
index 34328f4..31a0104 100644
--- a/target-arm/translate.h
+++ b/target-arm/translate.h
@@ -52,6 +52,11 @@
     return (dc->features & (1ULL << feature)) != 0;
 }
 
+static inline int get_mem_index(DisasContext *s)
+{
+    return s->current_pl;
+}
+
 /* target-specific extra values for is_jmp */
 /* These instructions trap after executing, so the A32/T32 decoder must
  * defer them until after the conditional execution state has been updated.
diff --git a/target-cris/helper.c b/target-cris/helper.c
index 4092d27..e8b8261 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -21,6 +21,7 @@
 #include "cpu.h"
 #include "mmu.h"
 #include "qemu/host-utils.h"
+#include "exec/cpu_ldst.h"
 
 
 //#define CRIS_HELPER_DEBUG
diff --git a/target-cris/helper.h b/target-cris/helper.h
index 0ac31f5..0b383b2 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_2(tlb_flush_pid, void, env, i32)
 DEF_HELPER_2(spc_write, void, env, i32)
@@ -25,5 +23,3 @@
 DEF_HELPER_FLAGS_3(evaluate_flags_move_2, TCG_CALL_NO_SE, i32, env, i32, i32)
 DEF_HELPER_1(evaluate_flags, void, env)
 DEF_HELPER_1(top_evaluate_flags, void, env)
-
-#include "exec/def-helper.h"
diff --git a/target-cris/op_helper.c b/target-cris/op_helper.c
index bd9a583..5c0c14d 100644
--- a/target-cris/op_helper.c
+++ b/target-cris/op_helper.c
@@ -20,8 +20,9 @@
 
 #include "cpu.h"
 #include "mmu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "exec/cpu_ldst.h"
 
 //#define CRIS_OP_HELPER_DEBUG
 
@@ -35,22 +36,6 @@
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 724f920..f26c323 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -26,12 +26,12 @@
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "mmu.h"
+#include "exec/cpu_ldst.h"
 #include "crisv32-decode.h"
 
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-gen.h"
 
 #define DISAS_CRIS 0
 #if DISAS_CRIS
diff --git a/target-i386/cc_helper.c b/target-i386/cc_helper.c
index 05dd12b..ecbf0ec 100644
--- a/target-i386/cc_helper.c
+++ b/target-i386/cc_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 const uint8_t parity_table[256] = {
     CC_P, 0, 0, CC_P, 0, CC_P, CC_P, 0,
diff --git a/target-i386/cpu-qom.h b/target-i386/cpu-qom.h
index e9b3d57..0808cfc 100644
--- a/target-i386/cpu-qom.h
+++ b/target-i386/cpu-qom.h
@@ -87,6 +87,7 @@
     bool hyperv_time;
     bool check_cpuid;
     bool enforce_cpuid;
+    bool expose_kvm;
 
     /* if true the CPUID code directly forward host cache leaves to the guest */
     bool cache_info_passthrough;
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 042a48d..dde052c 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -552,8 +552,7 @@
           CPUID_PSE36 | CPUID_CLFLUSH | CPUID_ACPI | CPUID_MMX | \
           CPUID_FXSR | CPUID_SSE | CPUID_SSE2 | CPUID_SS)
           /* partly implemented:
-          CPUID_MTRR, CPUID_MCA, CPUID_CLFLUSH (needed for Win64)
-          CPUID_PSE36 (needed for Solaris) */
+          CPUID_MTRR, CPUID_MCA, CPUID_CLFLUSH (needed for Win64) */
           /* missing:
           CPUID_VME, CPUID_DTS, CPUID_SS, CPUID_HT, CPUID_TM, CPUID_PBE */
 #define TCG_EXT_FEATURES (CPUID_EXT_SSE3 | CPUID_EXT_PCLMULQDQ | \
@@ -569,9 +568,7 @@
           CPUID_EXT_RDRAND */
 #define TCG_EXT2_FEATURES ((TCG_FEATURES & CPUID_EXT2_AMD_ALIASES) | \
           CPUID_EXT2_NX | CPUID_EXT2_MMXEXT | CPUID_EXT2_RDTSCP | \
-          CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT)
-          /* missing:
-          CPUID_EXT2_PDPE1GB */
+          CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_PDPE1GB)
 #define TCG_EXT3_FEATURES (CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM | \
           CPUID_EXT3_CR8LEG | CPUID_EXT3_ABM | CPUID_EXT3_SSE4A)
 #define TCG_SVM_FEATURES 0
@@ -2792,6 +2789,7 @@
     DEFINE_PROP_BOOL("hv-time", X86CPU, hyperv_time, false),
     DEFINE_PROP_BOOL("check", X86CPU, check_cpuid, false),
     DEFINE_PROP_BOOL("enforce", X86CPU, enforce_cpuid, false),
+    DEFINE_PROP_BOOL("kvm", X86CPU, expose_kvm, true),
     DEFINE_PROP_END_OF_LIST()
 };
 
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index e9cbdab..b5e1b41 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -249,6 +249,7 @@
 #define PG_DIRTY_BIT    6
 #define PG_PSE_BIT      7
 #define PG_GLOBAL_BIT   8
+#define PG_PSE_PAT_BIT  12
 #define PG_NX_BIT       63
 
 #define PG_PRESENT_MASK  (1 << PG_PRESENT_BIT)
@@ -260,6 +261,9 @@
 #define PG_DIRTY_MASK    (1 << PG_DIRTY_BIT)
 #define PG_PSE_MASK      (1 << PG_PSE_BIT)
 #define PG_GLOBAL_MASK   (1 << PG_GLOBAL_BIT)
+#define PG_PSE_PAT_MASK  (1 << PG_PSE_PAT_BIT)
+#define PG_ADDRESS_MASK  0x000ffffffffff000LL
+#define PG_HI_RSVD_MASK  (PG_ADDRESS_MASK & ~PHYS_ADDR_MASK)
 #define PG_HI_USER_MASK  0x7ff0000000000000LL
 #define PG_NX_MASK       (1LL << PG_NX_BIT)
 
@@ -986,7 +990,6 @@
     /* update the hidden flags */
     {
         if (seg_reg == R_CS) {
-            int cpl = selector & 3;
 #ifdef TARGET_X86_64
             if ((env->hflags & HF_LMA_MASK) && (flags & DESC_L_MASK)) {
                 /* long mode */
@@ -996,15 +999,14 @@
 #endif
             {
                 /* legacy / compatibility case */
-                if (!(env->cr[0] & CR0_PE_MASK))
-                    cpl = 0;
-                else if (env->eflags & VM_MASK)
-                    cpl = 3;
                 new_hflags = (env->segs[R_CS].flags & DESC_B_MASK)
                     >> (DESC_B_SHIFT - HF_CS32_SHIFT);
                 env->hflags = (env->hflags & ~(HF_CS32_MASK | HF_CS64_MASK)) |
                     new_hflags;
             }
+        }
+        if (seg_reg == R_SS) {
+            int cpl = (flags >> DESC_DPL_SHIFT) & 3;
 #if HF_CPL_MASK != 3
 #error HF_CPL_MASK is hardcoded
 #endif
@@ -1137,6 +1139,14 @@
 #define TARGET_VIRT_ADDR_SPACE_BITS 32
 #endif
 
+/* XXX: This value should match the one returned by CPUID
+ * and in exec.c */
+# if defined(TARGET_X86_64)
+# define PHYS_ADDR_MASK 0xffffffffffLL
+# else
+# define PHYS_ADDR_MASK 0xfffffffffLL
+# endif
+
 static inline CPUX86State *cpu_init(const char *cpu_model)
 {
     X86CPU *cpu = cpu_x86_init(cpu_model);
@@ -1153,17 +1163,24 @@
 #define cpudef_setup x86_cpudef_setup
 
 /* MMU modes definitions */
-#define MMU_MODE0_SUFFIX _kernel
+#define MMU_MODE0_SUFFIX _ksmap
 #define MMU_MODE1_SUFFIX _user
-#define MMU_MODE2_SUFFIX _ksmap /* Kernel with SMAP override */
-#define MMU_KERNEL_IDX  0
+#define MMU_MODE2_SUFFIX _knosmap /* SMAP disabled or CPL<3 && AC=1 */
+#define MMU_KSMAP_IDX   0
 #define MMU_USER_IDX    1
-#define MMU_KSMAP_IDX   2
-static inline int cpu_mmu_index (CPUX86State *env)
+#define MMU_KNOSMAP_IDX 2
+static inline int cpu_mmu_index(CPUX86State *env)
 {
     return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
-        ((env->hflags & HF_SMAP_MASK) && (env->eflags & AC_MASK))
-        ? MMU_KSMAP_IDX : MMU_KERNEL_IDX;
+        (!(env->hflags & HF_SMAP_MASK) || (env->eflags & AC_MASK))
+        ? MMU_KNOSMAP_IDX : MMU_KSMAP_IDX;
+}
+
+static inline int cpu_mmu_index_kernel(CPUX86State *env)
+{
+    return !(env->hflags & HF_SMAP_MASK) ? MMU_KNOSMAP_IDX :
+        ((env->hflags & HF_CPL_MASK) < 3 && (env->eflags & AC_MASK))
+        ? MMU_KNOSMAP_IDX : MMU_KSMAP_IDX;
 }
 
 #define CC_DST  (env->cc_dst)
@@ -1234,11 +1251,14 @@
     return env->eflags | cpu_cc_compute_all(env, CC_OP) | (env->df & DF_MASK);
 }
 
-/* NOTE: CC_OP must be modified manually to CC_OP_EFLAGS */
+/* NOTE: the translator must set DisasContext.cc_op to CC_OP_EFLAGS
+ * after generating a call to a helper that uses this.
+ */
 static inline void cpu_load_eflags(CPUX86State *env, int eflags,
                                    int update_mask)
 {
     CC_SRC = eflags & (CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
+    CC_OP = CC_OP_EFLAGS;
     env->df = 1 - (2 * ((eflags >> 10) & 1));
     env->eflags = (env->eflags & ~update_mask) |
         (eflags & update_mask) | 0x2;
diff --git a/target-i386/excp_helper.c b/target-i386/excp_helper.c
index f337fd2..99fca84 100644
--- a/target-i386/excp_helper.c
+++ b/target-i386/excp_helper.c
@@ -20,7 +20,7 @@
 #include "cpu.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #if 0
 #define raise_exception_err(env, a, b)                                  \
diff --git a/target-i386/fpu_helper.c b/target-i386/fpu_helper.c
index de7ba76..1b2900d 100644
--- a/target-i386/fpu_helper.c
+++ b/target-i386/fpu_helper.c
@@ -19,13 +19,10 @@
 
 #include <math.h>
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/aes.h"
 #include "qemu/host-utils.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
+#include "exec/cpu_ldst.h"
 
 #define FPU_RC_MASK         0xc00
 #define FPU_RC_NEAR         0x000
diff --git a/target-i386/gdbstub.c b/target-i386/gdbstub.c
index d34e535..19fe9ad 100644
--- a/target-i386/gdbstub.c
+++ b/target-i386/gdbstub.c
@@ -127,9 +127,11 @@
         target_ulong base;
 
         if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK)) {
+            int dpl = (env->eflags & VM_MASK) ? 3 : 0;
             base = selector << 4;
             limit = 0xffff;
-            flags = 0;
+            flags = DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                    DESC_A_MASK | (dpl << DESC_DPL_SHIFT);
         } else {
             if (!cpu_x86_get_descr_debug(env, selector, &base, &limit,
                                          &flags)) {
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 46d20e4..11ca864 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -510,14 +510,6 @@
 
 #else
 
-/* XXX: This value should match the one returned by CPUID
- * and in exec.c */
-# if defined(TARGET_X86_64)
-# define PHYS_ADDR_MASK 0xfffffff000LL
-# else
-# define PHYS_ADDR_MASK 0xffffff000LL
-# endif
-
 /* return value:
  * -1 = cannot handle fault
  * 0  = nothing more to do
@@ -530,10 +522,12 @@
     CPUX86State *env = &cpu->env;
     uint64_t ptep, pte;
     target_ulong pde_addr, pte_addr;
-    int error_code, is_dirty, prot, page_size, is_write, is_user;
+    int error_code = 0;
+    int is_dirty, prot, page_size, is_write, is_user;
     hwaddr paddr;
+    uint64_t rsvd_mask = PG_HI_RSVD_MASK;
     uint32_t page_offset;
-    target_ulong vaddr, virt_addr;
+    target_ulong vaddr;
 
     is_user = mmu_idx == MMU_USER_IDX;
 #if defined(DEBUG_MMU)
@@ -550,12 +544,15 @@
             pte = (uint32_t)pte;
         }
 #endif
-        virt_addr = addr & TARGET_PAGE_MASK;
         prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
         page_size = 4096;
         goto do_mapping;
     }
 
+    if (!(env->efer & MSR_EFER_NXE)) {
+        rsvd_mask |= PG_NX_MASK;
+    }
+
     if (env->cr[4] & CR4_PAE_MASK) {
         uint64_t pde, pdpe;
         target_ulong pdpe_addr;
@@ -577,34 +574,37 @@
                 env->a20_mask;
             pml4e = ldq_phys(cs->as, pml4e_addr);
             if (!(pml4e & PG_PRESENT_MASK)) {
-                error_code = 0;
                 goto do_fault;
             }
-            if (!(env->efer & MSR_EFER_NXE) && (pml4e & PG_NX_MASK)) {
-                error_code = PG_ERROR_RSVD_MASK;
-                goto do_fault;
+            if (pml4e & (rsvd_mask | PG_PSE_MASK)) {
+                goto do_fault_rsvd;
             }
             if (!(pml4e & PG_ACCESSED_MASK)) {
                 pml4e |= PG_ACCESSED_MASK;
                 stl_phys_notdirty(cs->as, pml4e_addr, pml4e);
             }
             ptep = pml4e ^ PG_NX_MASK;
-            pdpe_addr = ((pml4e & PHYS_ADDR_MASK) + (((addr >> 30) & 0x1ff) << 3)) &
+            pdpe_addr = ((pml4e & PG_ADDRESS_MASK) + (((addr >> 30) & 0x1ff) << 3)) &
                 env->a20_mask;
             pdpe = ldq_phys(cs->as, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK)) {
-                error_code = 0;
                 goto do_fault;
             }
-            if (!(env->efer & MSR_EFER_NXE) && (pdpe & PG_NX_MASK)) {
-                error_code = PG_ERROR_RSVD_MASK;
-                goto do_fault;
+            if (pdpe & rsvd_mask) {
+                goto do_fault_rsvd;
             }
             ptep &= pdpe ^ PG_NX_MASK;
             if (!(pdpe & PG_ACCESSED_MASK)) {
                 pdpe |= PG_ACCESSED_MASK;
                 stl_phys_notdirty(cs->as, pdpe_addr, pdpe);
             }
+            if (pdpe & PG_PSE_MASK) {
+                /* 1 GB page */
+                page_size = 1024 * 1024 * 1024;
+                pte_addr = pdpe_addr;
+                pte = pdpe;
+                goto do_check_protect;
+            }
         } else
 #endif
         {
@@ -613,134 +613,49 @@
                 env->a20_mask;
             pdpe = ldq_phys(cs->as, pdpe_addr);
             if (!(pdpe & PG_PRESENT_MASK)) {
-                error_code = 0;
                 goto do_fault;
             }
+            rsvd_mask |= PG_HI_USER_MASK | PG_NX_MASK;
+            if (pdpe & rsvd_mask) {
+                goto do_fault_rsvd;
+            }
             ptep = PG_NX_MASK | PG_USER_MASK | PG_RW_MASK;
         }
 
-        pde_addr = ((pdpe & PHYS_ADDR_MASK) + (((addr >> 21) & 0x1ff) << 3)) &
+        pde_addr = ((pdpe & PG_ADDRESS_MASK) + (((addr >> 21) & 0x1ff) << 3)) &
             env->a20_mask;
         pde = ldq_phys(cs->as, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
-            error_code = 0;
             goto do_fault;
         }
-        if (!(env->efer & MSR_EFER_NXE) && (pde & PG_NX_MASK)) {
-            error_code = PG_ERROR_RSVD_MASK;
-            goto do_fault;
+        if (pde & rsvd_mask) {
+            goto do_fault_rsvd;
         }
         ptep &= pde ^ PG_NX_MASK;
         if (pde & PG_PSE_MASK) {
             /* 2 MB page */
             page_size = 2048 * 1024;
-            ptep ^= PG_NX_MASK;
-            if ((ptep & PG_NX_MASK) && is_write1 == 2) {
-                goto do_fault_protect;
-            }
-            switch (mmu_idx) {
-            case MMU_USER_IDX:
-                if (!(ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if (is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            case MMU_KERNEL_IDX:
-                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                /* fall through */
-            case MMU_KSMAP_IDX:
-                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            default: /* cannot happen */
-                break;
-            }
-            is_dirty = is_write && !(pde & PG_DIRTY_MASK);
-            if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
-                pde |= PG_ACCESSED_MASK;
-                if (is_dirty)
-                    pde |= PG_DIRTY_MASK;
-                stl_phys_notdirty(cs->as, pde_addr, pde);
-            }
-            /* align to page_size */
-            pte = pde & ((PHYS_ADDR_MASK & ~(page_size - 1)) | 0xfff);
-            virt_addr = addr & ~(page_size - 1);
-        } else {
-            /* 4 KB page */
-            if (!(pde & PG_ACCESSED_MASK)) {
-                pde |= PG_ACCESSED_MASK;
-                stl_phys_notdirty(cs->as, pde_addr, pde);
-            }
-            pte_addr = ((pde & PHYS_ADDR_MASK) + (((addr >> 12) & 0x1ff) << 3)) &
-                env->a20_mask;
-            pte = ldq_phys(cs->as, pte_addr);
-            if (!(pte & PG_PRESENT_MASK)) {
-                error_code = 0;
-                goto do_fault;
-            }
-            if (!(env->efer & MSR_EFER_NXE) && (pte & PG_NX_MASK)) {
-                error_code = PG_ERROR_RSVD_MASK;
-                goto do_fault;
-            }
-            /* combine pde and pte nx, user and rw protections */
-            ptep &= pte ^ PG_NX_MASK;
-            ptep ^= PG_NX_MASK;
-            if ((ptep & PG_NX_MASK) && is_write1 == 2)
-                goto do_fault_protect;
-            switch (mmu_idx) {
-            case MMU_USER_IDX:
-                if (!(ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if (is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            case MMU_KERNEL_IDX:
-                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                /* fall through */
-            case MMU_KSMAP_IDX:
-                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            default: /* cannot happen */
-                break;
-            }
-            is_dirty = is_write && !(pte & PG_DIRTY_MASK);
-            if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
-                pte |= PG_ACCESSED_MASK;
-                if (is_dirty)
-                    pte |= PG_DIRTY_MASK;
-                stl_phys_notdirty(cs->as, pte_addr, pte);
-            }
-            page_size = 4096;
-            virt_addr = addr & ~0xfff;
-            pte = pte & (PHYS_ADDR_MASK | 0xfff);
+            pte_addr = pde_addr;
+            pte = pde;
+            goto do_check_protect;
         }
+        /* 4 KB page */
+        if (!(pde & PG_ACCESSED_MASK)) {
+            pde |= PG_ACCESSED_MASK;
+            stl_phys_notdirty(cs->as, pde_addr, pde);
+        }
+        pte_addr = ((pde & PG_ADDRESS_MASK) + (((addr >> 12) & 0x1ff) << 3)) &
+            env->a20_mask;
+        pte = ldq_phys(cs->as, pte_addr);
+        if (!(pte & PG_PRESENT_MASK)) {
+            goto do_fault;
+        }
+        if (pte & rsvd_mask) {
+            goto do_fault_rsvd;
+        }
+        /* combine pde and pte nx, user and rw protections */
+        ptep &= pte ^ PG_NX_MASK;
+        page_size = 4096;
     } else {
         uint32_t pde;
 
@@ -749,114 +664,95 @@
             env->a20_mask;
         pde = ldl_phys(cs->as, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
-            error_code = 0;
             goto do_fault;
         }
+        ptep = pde | PG_NX_MASK;
+
         /* if PSE bit is set, then we use a 4MB page */
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
             page_size = 4096 * 1024;
-            switch (mmu_idx) {
-            case MMU_USER_IDX:
-                if (!(pde & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if (is_write && !(pde & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
+            pte_addr = pde_addr;
 
-            case MMU_KERNEL_IDX:
-                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
-                    (pde & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                /* fall through */
-            case MMU_KSMAP_IDX:
-                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-                    (pde & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(pde & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            default: /* cannot happen */
-                break;
-            }
-            is_dirty = is_write && !(pde & PG_DIRTY_MASK);
-            if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
-                pde |= PG_ACCESSED_MASK;
-                if (is_dirty)
-                    pde |= PG_DIRTY_MASK;
-                stl_phys_notdirty(cs->as, pde_addr, pde);
-            }
-
-            pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */
-            ptep = pte;
-            virt_addr = addr & ~(page_size - 1);
-        } else {
-            if (!(pde & PG_ACCESSED_MASK)) {
-                pde |= PG_ACCESSED_MASK;
-                stl_phys_notdirty(cs->as, pde_addr, pde);
-            }
-
-            /* page directory entry */
-            pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) &
-                env->a20_mask;
-            pte = ldl_phys(cs->as, pte_addr);
-            if (!(pte & PG_PRESENT_MASK)) {
-                error_code = 0;
-                goto do_fault;
-            }
-            /* combine pde and pte user and rw protections */
-            ptep = pte & pde;
-            switch (mmu_idx) {
-            case MMU_USER_IDX:
-                if (!(ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if (is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            case MMU_KERNEL_IDX:
-                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                /* fall through */
-            case MMU_KSMAP_IDX:
-                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
-                    (ptep & PG_USER_MASK)) {
-                    goto do_fault_protect;
-                }
-                if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK)) {
-                    goto do_fault_protect;
-                }
-                break;
-
-            default: /* cannot happen */
-                break;
-            }
-            is_dirty = is_write && !(pte & PG_DIRTY_MASK);
-            if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
-                pte |= PG_ACCESSED_MASK;
-                if (is_dirty)
-                    pte |= PG_DIRTY_MASK;
-                stl_phys_notdirty(cs->as, pte_addr, pte);
-            }
-            page_size = 4096;
-            virt_addr = addr & ~0xfff;
+            /* Bits 20-13 provide bits 39-32 of the address, bit 21 is reserved.
+             * Leave bits 20-13 in place for setting accessed/dirty bits below.
+             */
+            pte = pde | ((pde & 0x1fe000) << (32 - 13));
+            rsvd_mask = 0x200000;
+            goto do_check_protect_pse36;
         }
+
+        if (!(pde & PG_ACCESSED_MASK)) {
+            pde |= PG_ACCESSED_MASK;
+            stl_phys_notdirty(cs->as, pde_addr, pde);
+        }
+
+        /* page directory entry */
+        pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) &
+            env->a20_mask;
+        pte = ldl_phys(cs->as, pte_addr);
+        if (!(pte & PG_PRESENT_MASK)) {
+            goto do_fault;
+        }
+        /* combine pde and pte user and rw protections */
+        ptep &= pte | PG_NX_MASK;
+        page_size = 4096;
+        rsvd_mask = 0;
     }
+
+do_check_protect:
+    rsvd_mask |= (page_size - 1) & PG_ADDRESS_MASK & ~PG_PSE_PAT_MASK;
+do_check_protect_pse36:
+    if (pte & rsvd_mask) {
+        goto do_fault_rsvd;
+    }
+    ptep ^= PG_NX_MASK;
+    if ((ptep & PG_NX_MASK) && is_write1 == 2) {
+        goto do_fault_protect;
+    }
+    switch (mmu_idx) {
+    case MMU_USER_IDX:
+        if (!(ptep & PG_USER_MASK)) {
+            goto do_fault_protect;
+        }
+        if (is_write && !(ptep & PG_RW_MASK)) {
+            goto do_fault_protect;
+        }
+        break;
+
+    case MMU_KSMAP_IDX:
+        if (is_write1 != 2 && (ptep & PG_USER_MASK)) {
+            goto do_fault_protect;
+        }
+        /* fall through */
+    case MMU_KNOSMAP_IDX:
+        if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+            (ptep & PG_USER_MASK)) {
+            goto do_fault_protect;
+        }
+        if ((env->cr[0] & CR0_WP_MASK) &&
+            is_write && !(ptep & PG_RW_MASK)) {
+            goto do_fault_protect;
+        }
+        break;
+
+    default: /* cannot happen */
+        break;
+    }
+    is_dirty = is_write && !(pte & PG_DIRTY_MASK);
+    if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
+        pte |= PG_ACCESSED_MASK;
+        if (is_dirty) {
+            pte |= PG_DIRTY_MASK;
+        }
+        stl_phys_notdirty(cs->as, pte_addr, pte);
+    }
+
     /* the page can be put in the TLB */
     prot = PAGE_READ;
-    if (!(ptep & PG_NX_MASK))
+    if (!(ptep & PG_NX_MASK) &&
+        !((env->cr[4] & CR4_SMEP_MASK) && (ptep & PG_USER_MASK))) {
         prot |= PAGE_EXEC;
+    }
     if (pte & PG_DIRTY_MASK) {
         /* only set write access if already dirty... otherwise wait
            for dirty access */
@@ -872,16 +768,21 @@
  do_mapping:
     pte = pte & env->a20_mask;
 
+    /* align to page_size */
+    pte &= PG_ADDRESS_MASK & ~(page_size - 1);
+
     /* Even if 4MB pages, we map only one 4KB page in the cache to
        avoid filling it too fast */
-    page_offset = (addr & TARGET_PAGE_MASK) & (page_size - 1);
-    paddr = (pte & TARGET_PAGE_MASK) + page_offset;
-    vaddr = virt_addr + page_offset;
+    vaddr = addr & TARGET_PAGE_MASK;
+    page_offset = vaddr & (page_size - 1);
+    paddr = pte + page_offset;
 
     tlb_set_page(cs, vaddr, paddr, prot, mmu_idx, page_size);
     return 0;
+ do_fault_rsvd:
+    error_code |= PG_ERROR_RSVD_MASK;
  do_fault_protect:
-    error_code = PG_ERROR_P_MASK;
+    error_code |= PG_ERROR_P_MASK;
  do_fault:
     error_code |= (is_write << PG_ERROR_W_BIT);
     if (is_user)
@@ -910,7 +811,6 @@
     CPUX86State *env = &cpu->env;
     target_ulong pde_addr, pte_addr;
     uint64_t pte;
-    hwaddr paddr;
     uint32_t page_offset;
     int page_size;
 
@@ -928,25 +828,24 @@
 
             /* test virtual address sign extension */
             sext = (int64_t)addr >> 47;
-            if (sext != 0 && sext != -1)
+            if (sext != 0 && sext != -1) {
                 return -1;
-
+            }
             pml4e_addr = ((env->cr[3] & ~0xfff) + (((addr >> 39) & 0x1ff) << 3)) &
                 env->a20_mask;
             pml4e = ldq_phys(cs->as, pml4e_addr);
-            if (!(pml4e & PG_PRESENT_MASK))
+            if (!(pml4e & PG_PRESENT_MASK)) {
                 return -1;
-
-            pdpe_addr = ((pml4e & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) +
+            }
+            pdpe_addr = ((pml4e & PG_ADDRESS_MASK) +
                          (((addr >> 30) & 0x1ff) << 3)) & env->a20_mask;
             pdpe = ldq_phys(cs->as, pdpe_addr);
-            if (!(pdpe & PG_PRESENT_MASK))
+            if (!(pdpe & PG_PRESENT_MASK)) {
                 return -1;
-
+            }
             if (pdpe & PG_PSE_MASK) {
                 page_size = 1024 * 1024 * 1024;
-                pte = pdpe & ~( (page_size - 1) & ~0xfff);
-                pte &= ~(PG_NX_MASK | PG_HI_USER_MASK);
+                pte = pdpe;
                 goto out;
             }
 
@@ -960,7 +859,7 @@
                 return -1;
         }
 
-        pde_addr = ((pdpe & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) +
+        pde_addr = ((pdpe & PG_ADDRESS_MASK) +
                     (((addr >> 21) & 0x1ff) << 3)) & env->a20_mask;
         pde = ldq_phys(cs->as, pde_addr);
         if (!(pde & PG_PRESENT_MASK)) {
@@ -969,17 +868,17 @@
         if (pde & PG_PSE_MASK) {
             /* 2 MB page */
             page_size = 2048 * 1024;
-            pte = pde & ~( (page_size - 1) & ~0xfff); /* align to page_size */
+            pte = pde;
         } else {
             /* 4 KB page */
-            pte_addr = ((pde & ~0xfff & ~(PG_NX_MASK | PG_HI_USER_MASK)) +
+            pte_addr = ((pde & PG_ADDRESS_MASK) +
                         (((addr >> 12) & 0x1ff) << 3)) & env->a20_mask;
             page_size = 4096;
             pte = ldq_phys(cs->as, pte_addr);
         }
-        pte &= ~(PG_NX_MASK | PG_HI_USER_MASK);
-        if (!(pte & PG_PRESENT_MASK))
+        if (!(pte & PG_PRESENT_MASK)) {
             return -1;
+        }
     } else {
         uint32_t pde;
 
@@ -989,14 +888,15 @@
         if (!(pde & PG_PRESENT_MASK))
             return -1;
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
-            pte = pde & ~0x003ff000; /* align to 4MB */
+            pte = pde | ((pde & 0x1fe000) << (32 - 13));
             page_size = 4096 * 1024;
         } else {
             /* page directory entry */
             pte_addr = ((pde & ~0xfff) + ((addr >> 10) & 0xffc)) & env->a20_mask;
             pte = ldl_phys(cs->as, pte_addr);
-            if (!(pte & PG_PRESENT_MASK))
+            if (!(pte & PG_PRESENT_MASK)) {
                 return -1;
+            }
             page_size = 4096;
         }
         pte = pte & env->a20_mask;
@@ -1005,9 +905,9 @@
 #ifdef TARGET_X86_64
 out:
 #endif
+    pte &= PG_ADDRESS_MASK & ~(page_size - 1);
     page_offset = (addr & TARGET_PAGE_MASK) & (page_size - 1);
-    paddr = (pte & TARGET_PAGE_MASK) + page_offset;
-    return paddr;
+    return pte | page_offset;
 }
 
 void hw_breakpoint_insert(CPUX86State *env, int index)
diff --git a/target-i386/helper.h b/target-i386/helper.h
index 3775abe..8eb0145 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_FLAGS_4(cc_compute_all, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl, int)
 DEF_HELPER_FLAGS_4(cc_compute_c, TCG_CALL_NO_RWG_SE, tl, tl, tl, tl, int)
 
@@ -219,5 +217,3 @@
 DEF_HELPER_3(rclq, tl, env, tl, tl)
 DEF_HELPER_3(rcrq, tl, env, tl, tl)
 #endif
-
-#include "exec/def-helper.h"
diff --git a/target-i386/int_helper.c b/target-i386/int_helper.c
index 0555318..b0d78e6 100644
--- a/target-i386/int_helper.c
+++ b/target-i386/int_helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 //#define DEBUG_MULDIV
 
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index 0d894ef..4bf0ac9 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -528,23 +528,25 @@
         has_msr_hv_hypercall = true;
     }
 
-    memcpy(signature, "KVMKVMKVM\0\0\0", 12);
-    c = &cpuid_data.entries[cpuid_i++];
-    c->function = KVM_CPUID_SIGNATURE | kvm_base;
-    c->eax = 0;
-    c->ebx = signature[0];
-    c->ecx = signature[1];
-    c->edx = signature[2];
+    if (cpu->expose_kvm) {
+        memcpy(signature, "KVMKVMKVM\0\0\0", 12);
+        c = &cpuid_data.entries[cpuid_i++];
+        c->function = KVM_CPUID_SIGNATURE | kvm_base;
+        c->eax = KVM_CPUID_FEATURES | kvm_base;
+        c->ebx = signature[0];
+        c->ecx = signature[1];
+        c->edx = signature[2];
 
-    c = &cpuid_data.entries[cpuid_i++];
-    c->function = KVM_CPUID_FEATURES | kvm_base;
-    c->eax = env->features[FEAT_KVM];
+        c = &cpuid_data.entries[cpuid_i++];
+        c->function = KVM_CPUID_FEATURES | kvm_base;
+        c->eax = env->features[FEAT_KVM];
 
-    has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
+        has_msr_async_pf_en = c->eax & (1 << KVM_FEATURE_ASYNC_PF);
 
-    has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
+        has_msr_pv_eoi_en = c->eax & (1 << KVM_FEATURE_PV_EOI);
 
-    has_msr_kvm_steal_time = c->eax & (1 << KVM_FEATURE_STEAL_TIME);
+        has_msr_kvm_steal_time = c->eax & (1 << KVM_FEATURE_STEAL_TIME);
+    }
 
     cpu_x86_cpuid(env, 0, 0, &limit, &unused, &unused, &unused);
 
@@ -1430,7 +1432,7 @@
        HF_OSFXSR_MASK | HF_LMA_MASK | HF_CS32_MASK | \
        HF_SS32_MASK | HF_CS64_MASK | HF_ADDSEG_MASK)
 
-    hflags = (env->segs[R_CS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
+    hflags = (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
     hflags |= (env->cr[0] & CR0_PE_MASK) << (HF_PE_SHIFT - CR0_PE_SHIFT);
     hflags |= (env->cr[0] << (HF_MP_SHIFT - CR0_MP_SHIFT)) &
                 (HF_MP_MASK | HF_EM_MASK | HF_TS_MASK);
diff --git a/target-i386/machine.c b/target-i386/machine.c
index 168cab6..bdff447 100644
--- a/target-i386/machine.c
+++ b/target-i386/machine.c
@@ -312,6 +312,14 @@
         env->segs[R_SS].flags &= ~(env->segs[R_SS].flags & DESC_DPL_MASK);
     }
 
+    /* Older versions of QEMU incorrectly used CS.DPL as the CPL when
+     * running under KVM.  This is wrong for conforming code segments.
+     * Luckily, in our implementation the CPL field of hflags is redundant
+     * and we can get the right value from the SS descriptor privilege level.
+     */
+    env->hflags &= ~HF_CPL_MASK;
+    env->hflags |= (env->segs[R_SS].flags >> DESC_DPL_SHIFT) & HF_CPL_MASK;
+
     /* XXX: restore FPU round state */
     env->fpstt = (env->fpus_vmstate >> 11) & 7;
     env->fpus = env->fpus_vmstate & ~0x3800;
diff --git a/target-i386/mem_helper.c b/target-i386/mem_helper.c
index b3b811b..1aec8a5 100644
--- a/target-i386/mem_helper.c
+++ b/target-i386/mem_helper.c
@@ -18,11 +18,8 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 /* broken thread support */
 
@@ -110,24 +107,6 @@
 }
 
 #if !defined(CONFIG_USER_ONLY)
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
-#endif
-
-#if !defined(CONFIG_USER_ONLY)
 /* try to fill the TLB and return an exception if error. If retaddr is
  * NULL, it means that the function was called in C code (i.e. not
  * from generated code or from helper.c)
diff --git a/target-i386/misc_helper.c b/target-i386/misc_helper.c
index 1e2da1e..4aaf1e4 100644
--- a/target-i386/misc_helper.c
+++ b/target-i386/misc_helper.c
@@ -19,53 +19,8 @@
 
 #include "cpu.h"
 #include "exec/ioport.h"
-#include "helper.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
-
-/* check if Port I/O is allowed in TSS */
-static inline void check_io(CPUX86State *env, int addr, int size)
-{
-    int io_offset, val, mask;
-
-    /* TSS must be a valid 32 bit one */
-    if (!(env->tr.flags & DESC_P_MASK) ||
-        ((env->tr.flags >> DESC_TYPE_SHIFT) & 0xf) != 9 ||
-        env->tr.limit < 103) {
-        goto fail;
-    }
-    io_offset = cpu_lduw_kernel(env, env->tr.base + 0x66);
-    io_offset += (addr >> 3);
-    /* Note: the check needs two bytes */
-    if ((io_offset + 1) > env->tr.limit) {
-        goto fail;
-    }
-    val = cpu_lduw_kernel(env, env->tr.base + io_offset);
-    val >>= (addr & 7);
-    mask = (1 << size) - 1;
-    /* all bits must be zero to allow the I/O */
-    if ((val & mask) != 0) {
-    fail:
-        raise_exception_err(env, EXCP0D_GPF, 0);
-    }
-}
-
-void helper_check_iob(CPUX86State *env, uint32_t t0)
-{
-    check_io(env, t0, 1);
-}
-
-void helper_check_iow(CPUX86State *env, uint32_t t0)
-{
-    check_io(env, t0, 2);
-}
-
-void helper_check_iol(CPUX86State *env, uint32_t t0)
-{
-    check_io(env, t0, 4);
-}
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 void helper_outb(uint32_t port, uint32_t data)
 {
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 3cf862e..2d970d0 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -20,14 +20,11 @@
 
 #include "cpu.h"
 #include "qemu/log.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 //#define DEBUG_PCALL
 
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
-
 #ifdef DEBUG_PCALL
 # define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
 # define LOG_PCALL_STATE(cpu)                                  \
@@ -37,6 +34,24 @@
 # define LOG_PCALL_STATE(cpu) do { } while (0)
 #endif
 
+#ifndef CONFIG_USER_ONLY
+#define CPU_MMU_INDEX (cpu_mmu_index_kernel(env))
+#define MEMSUFFIX _kernel
+#define DATA_SIZE 1
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 2
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 4
+#include "exec/cpu_ldst_template.h"
+
+#define DATA_SIZE 8
+#include "exec/cpu_ldst_template.h"
+#undef CPU_MMU_INDEX
+#undef MEMSUFFIX
+#endif
+
 /* return non zero if error */
 static inline int load_segment(CPUX86State *env, uint32_t *e1_ptr,
                                uint32_t *e2_ptr, int selector)
@@ -88,8 +103,10 @@
 static inline void load_seg_vm(CPUX86State *env, int seg, int selector)
 {
     selector &= 0xffff;
-    cpu_x86_load_seg_cache(env, seg, selector,
-                           (selector << 4), 0xffff, 0);
+
+    cpu_x86_load_seg_cache(env, seg, selector, (selector << 4), 0xffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK | (3 << DESC_DPL_SHIFT));
 }
 
 static inline void get_ss_esp_from_tss(CPUX86State *env, uint32_t *ss_ptr,
@@ -133,11 +150,10 @@
     }
 }
 
-/* XXX: merge with load_seg() */
-static void tss_load_seg(CPUX86State *env, int seg_reg, int selector)
+static void tss_load_seg(CPUX86State *env, int seg_reg, int selector, int cpl)
 {
     uint32_t e1, e2;
-    int rpl, dpl, cpl;
+    int rpl, dpl;
 
     if ((selector & 0xfffc) != 0) {
         if (load_segment(env, &e1, &e2, selector) != 0) {
@@ -148,18 +164,13 @@
         }
         rpl = selector & 3;
         dpl = (e2 >> DESC_DPL_SHIFT) & 3;
-        cpl = env->hflags & HF_CPL_MASK;
         if (seg_reg == R_CS) {
             if (!(e2 & DESC_CS_MASK)) {
                 raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
             }
-            /* XXX: is it correct? */
             if (dpl != rpl) {
                 raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
             }
-            if ((e2 & DESC_C_MASK) && dpl > rpl) {
-                raise_exception_err(env, EXCP0A_TSS, selector & 0xfffc);
-            }
         } else if (seg_reg == R_SS) {
             /* SS must be writable data */
             if ((e2 & DESC_CS_MASK) || !(e2 & DESC_W_MASK)) {
@@ -446,12 +457,13 @@
 
     /* load the segments */
     if (!(new_eflags & VM_MASK)) {
-        tss_load_seg(env, R_CS, new_segs[R_CS]);
-        tss_load_seg(env, R_SS, new_segs[R_SS]);
-        tss_load_seg(env, R_ES, new_segs[R_ES]);
-        tss_load_seg(env, R_DS, new_segs[R_DS]);
-        tss_load_seg(env, R_FS, new_segs[R_FS]);
-        tss_load_seg(env, R_GS, new_segs[R_GS]);
+        int cpl = new_segs[R_CS] & 3;
+        tss_load_seg(env, R_CS, new_segs[R_CS], cpl);
+        tss_load_seg(env, R_SS, new_segs[R_SS], cpl);
+        tss_load_seg(env, R_ES, new_segs[R_ES], cpl);
+        tss_load_seg(env, R_DS, new_segs[R_DS], cpl);
+        tss_load_seg(env, R_FS, new_segs[R_FS], cpl);
+        tss_load_seg(env, R_GS, new_segs[R_GS], cpl);
     }
 
     /* check that env->eip is in the CS segment limits */
@@ -558,6 +570,7 @@
     int has_error_code, new_stack, shift;
     uint32_t e1, e2, offset, ss = 0, esp, ss_e1 = 0, ss_e2 = 0;
     uint32_t old_eip, sp_mask;
+    int vm86 = env->eflags & VM_MASK;
 
     has_error_code = 0;
     if (!is_int && !is_hw) {
@@ -673,7 +686,7 @@
         ssp = get_seg_base(ss_e1, ss_e2);
     } else if ((e2 & DESC_C_MASK) || dpl == cpl) {
         /* to same privilege */
-        if (env->eflags & VM_MASK) {
+        if (vm86) {
             raise_exception_err(env, EXCP0D_GPF, selector & 0xfffc);
         }
         new_stack = 0;
@@ -694,14 +707,14 @@
 #if 0
     /* XXX: check that enough room is available */
     push_size = 6 + (new_stack << 2) + (has_error_code << 1);
-    if (env->eflags & VM_MASK) {
+    if (vm86) {
         push_size += 8;
     }
     push_size <<= shift;
 #endif
     if (shift == 1) {
         if (new_stack) {
-            if (env->eflags & VM_MASK) {
+            if (vm86) {
                 PUSHL(ssp, esp, sp_mask, env->segs[R_GS].selector);
                 PUSHL(ssp, esp, sp_mask, env->segs[R_FS].selector);
                 PUSHL(ssp, esp, sp_mask, env->segs[R_DS].selector);
@@ -718,7 +731,7 @@
         }
     } else {
         if (new_stack) {
-            if (env->eflags & VM_MASK) {
+            if (vm86) {
                 PUSHW(ssp, esp, sp_mask, env->segs[R_GS].selector);
                 PUSHW(ssp, esp, sp_mask, env->segs[R_FS].selector);
                 PUSHW(ssp, esp, sp_mask, env->segs[R_DS].selector);
@@ -742,7 +755,7 @@
     env->eflags &= ~(TF_MASK | VM_MASK | RF_MASK | NT_MASK);
 
     if (new_stack) {
-        if (env->eflags & VM_MASK) {
+        if (vm86) {
             cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0, 0);
             cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0, 0);
             cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0, 0);
@@ -1600,7 +1613,6 @@
             }
             next_eip = env->eip + next_eip_addend;
             switch_tss(env, new_cs, e1, e2, SWITCH_TSS_JMP, next_eip);
-            CC_OP = CC_OP_EFLAGS;
             break;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
@@ -1769,7 +1781,6 @@
                 raise_exception_err(env, EXCP0D_GPF, new_cs & 0xfffc);
             }
             switch_tss(env, new_cs, e1, e2, SWITCH_TSS_CALL, next_eip);
-            CC_OP = CC_OP_EFLAGS;
             return;
         case 4: /* 286 call gate */
         case 12: /* 386 call gate */
@@ -2464,11 +2475,56 @@
 void cpu_x86_load_seg(CPUX86State *env, int seg_reg, int selector)
 {
     if (!(env->cr[0] & CR0_PE_MASK) || (env->eflags & VM_MASK)) {
+        int dpl = (env->eflags & VM_MASK) ? 3 : 0;
         selector &= 0xffff;
         cpu_x86_load_seg_cache(env, seg_reg, selector,
-                               (selector << 4), 0xffff, 0);
+                               (selector << 4), 0xffff,
+                               DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                               DESC_A_MASK | (dpl << DESC_DPL_SHIFT));
     } else {
         helper_load_seg(env, seg_reg, selector);
     }
 }
 #endif
+
+/* check if Port I/O is allowed in TSS */
+static inline void check_io(CPUX86State *env, int addr, int size)
+{
+    int io_offset, val, mask;
+
+    /* TSS must be a valid 32 bit one */
+    if (!(env->tr.flags & DESC_P_MASK) ||
+        ((env->tr.flags >> DESC_TYPE_SHIFT) & 0xf) != 9 ||
+        env->tr.limit < 103) {
+        goto fail;
+    }
+    io_offset = cpu_lduw_kernel(env, env->tr.base + 0x66);
+    io_offset += (addr >> 3);
+    /* Note: the check needs two bytes */
+    if ((io_offset + 1) > env->tr.limit) {
+        goto fail;
+    }
+    val = cpu_lduw_kernel(env, env->tr.base + io_offset);
+    val >>= (addr & 7);
+    mask = (1 << size) - 1;
+    /* all bits must be zero to allow the I/O */
+    if ((val & mask) != 0) {
+    fail:
+        raise_exception_err(env, EXCP0D_GPF, 0);
+    }
+}
+
+void helper_check_iob(CPUX86State *env, uint32_t t0)
+{
+    check_io(env, t0, 1);
+}
+
+void helper_check_iow(CPUX86State *env, uint32_t t0)
+{
+    check_io(env, t0, 2);
+}
+
+void helper_check_iol(CPUX86State *env, uint32_t t0)
+{
+    check_io(env, t0, 4);
+}
diff --git a/target-i386/smm_helper.c b/target-i386/smm_helper.c
index 4841d53..58051d3 100644
--- a/target-i386/smm_helper.c
+++ b/target-i386/smm_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /* SMM support */
 
@@ -168,15 +168,26 @@
                                       CR0_PG_MASK));
     cpu_x86_update_cr4(env, 0);
     env->dr[7] = 0x00000400;
-    CC_OP = CC_OP_EFLAGS;
 
     cpu_x86_load_seg_cache(env, R_CS, (env->smbase >> 4) & 0xffff, env->smbase,
-                           0xffffffff, 0);
-    cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffffffff, 0);
-    cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffffffff, 0);
-    cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffffffff, 0);
-    cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffffffff, 0);
-    cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffffffff, 0);
+                           0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
+    cpu_x86_load_seg_cache(env, R_DS, 0, 0, 0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
+    cpu_x86_load_seg_cache(env, R_ES, 0, 0, 0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
+    cpu_x86_load_seg_cache(env, R_SS, 0, 0, 0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
+    cpu_x86_load_seg_cache(env, R_FS, 0, 0, 0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
+    cpu_x86_load_seg_cache(env, R_GS, 0, 0, 0xffffffff,
+                           DESC_P_MASK | DESC_S_MASK | DESC_W_MASK |
+                           DESC_A_MASK);
 }
 
 void helper_rsm(CPUX86State *env)
@@ -296,7 +307,6 @@
         env->smbase = ldl_phys(cs->as, sm_state + 0x7ef8) & ~0x7fff;
     }
 #endif
-    CC_OP = CC_OP_EFLAGS;
     env->hflags &= ~HF_SMM_MASK;
     cpu_smm_update(env);
 
diff --git a/target-i386/svm_helper.c b/target-i386/svm_helper.c
index 846eaa5..429d029 100644
--- a/target-i386/svm_helper.c
+++ b/target-i386/svm_helper.c
@@ -19,11 +19,8 @@
 
 #include "cpu.h"
 #include "exec/cpu-all.h"
-#include "helper.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 /* Secure Virtual Machine helpers */
 
@@ -260,7 +257,6 @@
                                   env->vm_vmcb + offsetof(struct vmcb,
                                                           save.rflags)),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK));
-    CC_OP = CC_OP_EFLAGS;
 
     svm_load_seg_cache(env, env->vm_vmcb + offsetof(struct vmcb, save.es),
                        R_ES);
@@ -702,7 +698,6 @@
                                                            save.rflags)),
                     ~(CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C | DF_MASK |
                       VM_MASK));
-    CC_OP = CC_OP_EFLAGS;
 
     svm_load_seg_cache(env, env->vm_hsave + offsetof(struct vmcb, save.es),
                        R_ES);
diff --git a/target-i386/translate.c b/target-i386/translate.c
index 032b0fd..2359787 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -27,10 +27,10 @@
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #define PREFIX_REPZ   0x01
 #define PREFIX_REPNZ  0x02
diff --git a/target-lm32/Makefile.objs b/target-lm32/Makefile.objs
index 4023687..c3e1bd6 100644
--- a/target-lm32/Makefile.objs
+++ b/target-lm32/Makefile.objs
@@ -1,3 +1,4 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += gdbstub.o
+obj-y += lm32-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
diff --git a/target-lm32/README b/target-lm32/README
index a1c2c7e..ba3508a 100644
--- a/target-lm32/README
+++ b/target-lm32/README
@@ -16,14 +16,13 @@
 available as virtual consoles.
 
 
-Programmatically terminate the emulator
-----------------------------------------
-Originally neither the LatticeMico32 nor its peripherals support a
-mechanism to shut down the machine. Emulation aware programs can write to a
-to a special register within the system control block to shut down the
-virtual machine.  For more details see hw/lm32_sys.c. The lm32-evr is the
-first BSP which instantiate this model. A (32 bit) write to 0xfff0000
-causes a vm shutdown.
+Semihosting
+-----------
+Semihosting on this target is supported. Some system calls like read, write
+and exit are executed on the host if semihosting is enabled. See
+target/lm32-semi.c for all supported system calls. Emulation aware programs
+can use this mechanism to shut down the virtual machine and print to the
+host console. See the tcg tests for an example.
 
 
 Special instructions
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index 24bde78..70600aa 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -217,6 +217,7 @@
 void lm32_watchpoint_insert(CPULM32State *env, int index, target_ulong address,
         lm32_wp_t wp_type);
 void lm32_watchpoint_remove(CPULM32State *env, int index);
+bool lm32_cpu_do_semihosting(CPUState *cs);
 
 static inline CPULM32State *cpu_init(const char *cpu_model)
 {
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 783aa16..1bca196 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -1,7 +1,7 @@
 /*
  *  LatticeMico32 helper routines.
  *
- *  Copyright (c) 2010 Michael Walle <michael@walle.cc>
+ *  Copyright (c) 2010-2014 Michael Walle <michael@walle.cc>
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -19,6 +19,7 @@
 
 #include "cpu.h"
 #include "qemu/host-utils.h"
+#include "sysemu/sysemu.h"
 
 int lm32_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
                               int mmu_idx)
@@ -159,11 +160,20 @@
             "exception at pc=%x type=%x\n", env->pc, cs->exception_index);
 
     switch (cs->exception_index) {
+    case EXCP_SYSTEMCALL:
+        if (unlikely(semihosting_enabled)) {
+            /* do_semicall() returns true if call was handled. Otherwise
+             * do the normal exception handling. */
+            if (lm32_cpu_do_semihosting(cs)) {
+                env->pc += 4;
+                break;
+            }
+        }
+        /* fall through */
     case EXCP_INSN_BUS_ERROR:
     case EXCP_DATA_BUS_ERROR:
     case EXCP_DIVIDE_BY_ZERO:
     case EXCP_IRQ:
-    case EXCP_SYSTEMCALL:
         /* non-debug exceptions */
         env->regs[R_EA] = env->pc;
         env->ie |= (env->ie & IE_IE) ? IE_EIE : 0;
diff --git a/target-lm32/helper.h b/target-lm32/helper.h
index f4442e0..445578c 100644
--- a/target-lm32/helper.h
+++ b/target-lm32/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_1(hlt, void, env)
 DEF_HELPER_3(wcsr_bp, void, env, i32, i32)
@@ -14,5 +12,3 @@
 DEF_HELPER_1(rcsr_jtx, i32, env)
 DEF_HELPER_1(rcsr_jrx, i32, env)
 DEF_HELPER_1(ill, void, env)
-
-#include "exec/def-helper.h"
diff --git a/target-lm32/lm32-semi.c b/target-lm32/lm32-semi.c
new file mode 100644
index 0000000..ec6524f
--- /dev/null
+++ b/target-lm32/lm32-semi.c
@@ -0,0 +1,215 @@
+/*
+ *  Lattice Mico32 semihosting syscall interface
+ *
+ *  Copyright (c) 2014 Michael Walle <michael@walle.cc>
+ *
+ * Based on target-m68k/m68k-semi.c, which is
+ *  Copyright (c) 2005-2007 CodeSourcery.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+#include <stddef.h>
+#include "cpu.h"
+#include "exec/helper-proto.h"
+#include "qemu/log.h"
+#include "exec/softmmu-semi.h"
+
+enum {
+    TARGET_SYS_exit    = 1,
+    TARGET_SYS_open    = 2,
+    TARGET_SYS_close   = 3,
+    TARGET_SYS_read    = 4,
+    TARGET_SYS_write   = 5,
+    TARGET_SYS_lseek   = 6,
+    TARGET_SYS_fstat   = 10,
+    TARGET_SYS_stat    = 15,
+};
+
+enum {
+    NEWLIB_O_RDONLY    =   0x0,
+    NEWLIB_O_WRONLY    =   0x1,
+    NEWLIB_O_RDWR      =   0x2,
+    NEWLIB_O_APPEND    =   0x8,
+    NEWLIB_O_CREAT     = 0x200,
+    NEWLIB_O_TRUNC     = 0x400,
+    NEWLIB_O_EXCL      = 0x800,
+};
+
+static int translate_openflags(int flags)
+{
+    int hf;
+
+    if (flags & NEWLIB_O_WRONLY) {
+        hf = O_WRONLY;
+    } else if (flags & NEWLIB_O_RDWR) {
+        hf = O_RDWR;
+    } else {
+        hf = O_RDONLY;
+    }
+
+    if (flags & NEWLIB_O_APPEND) {
+        hf |= O_APPEND;
+    }
+
+    if (flags & NEWLIB_O_CREAT) {
+        hf |= O_CREAT;
+    }
+
+    if (flags & NEWLIB_O_TRUNC) {
+        hf |= O_TRUNC;
+    }
+
+    if (flags & NEWLIB_O_EXCL) {
+        hf |= O_EXCL;
+    }
+
+    return hf;
+}
+
+struct newlib_stat {
+    int16_t     newlib_st_dev;     /* device */
+    uint16_t    newlib_st_ino;     /* inode */
+    uint16_t    newlib_st_mode;    /* protection */
+    uint16_t    newlib_st_nlink;   /* number of hard links */
+    uint16_t    newlib_st_uid;     /* user ID of owner */
+    uint16_t    newlib_st_gid;     /* group ID of owner */
+    int16_t     newlib_st_rdev;    /* device type (if inode device) */
+    int32_t     newlib_st_size;    /* total size, in bytes */
+    int32_t     newlib_st_atime;   /* time of last access */
+    uint32_t    newlib_st_spare1;
+    int32_t     newlib_st_mtime;   /* time of last modification */
+    uint32_t    newlib_st_spare2;
+    int32_t     newlib_st_ctime;   /* time of last change */
+    uint32_t    newlib_st_spare3;
+} QEMU_PACKED;
+
+static int translate_stat(CPULM32State *env, target_ulong addr,
+        struct stat *s)
+{
+    struct newlib_stat *p;
+
+    p = lock_user(VERIFY_WRITE, addr, sizeof(struct newlib_stat), 0);
+    if (!p) {
+        return 0;
+    }
+    p->newlib_st_dev = cpu_to_be16(s->st_dev);
+    p->newlib_st_ino = cpu_to_be16(s->st_ino);
+    p->newlib_st_mode = cpu_to_be16(s->st_mode);
+    p->newlib_st_nlink = cpu_to_be16(s->st_nlink);
+    p->newlib_st_uid = cpu_to_be16(s->st_uid);
+    p->newlib_st_gid = cpu_to_be16(s->st_gid);
+    p->newlib_st_rdev = cpu_to_be16(s->st_rdev);
+    p->newlib_st_size = cpu_to_be32(s->st_size);
+    p->newlib_st_atime = cpu_to_be32(s->st_atime);
+    p->newlib_st_mtime = cpu_to_be32(s->st_mtime);
+    p->newlib_st_ctime = cpu_to_be32(s->st_ctime);
+    unlock_user(p, addr, sizeof(struct newlib_stat));
+
+    return 1;
+}
+
+bool lm32_cpu_do_semihosting(CPUState *cs)
+{
+    LM32CPU *cpu = LM32_CPU(cs);
+    CPULM32State *env = &cpu->env;
+
+    int ret = -1;
+    target_ulong nr, arg0, arg1, arg2;
+    void *p;
+    struct stat s;
+
+    nr = env->regs[R_R8];
+    arg0 = env->regs[R_R1];
+    arg1 = env->regs[R_R2];
+    arg2 = env->regs[R_R3];
+
+    switch (nr) {
+    case TARGET_SYS_exit:
+        /* void _exit(int rc) */
+        exit(arg0);
+
+    case TARGET_SYS_open:
+        /* int open(const char *pathname, int flags) */
+        p = lock_user_string(arg0);
+        if (!p) {
+            ret = -1;
+        } else {
+            ret = open(p, translate_openflags(arg2));
+            unlock_user(p, arg0, 0);
+        }
+        break;
+
+    case TARGET_SYS_read:
+        /* ssize_t read(int fd, const void *buf, size_t count) */
+        p = lock_user(VERIFY_WRITE, arg1, arg2, 0);
+        if (!p) {
+            ret = -1;
+        } else {
+            ret = read(arg0, p, arg2);
+            unlock_user(p, arg1, arg2);
+        }
+        break;
+
+    case TARGET_SYS_write:
+        /* ssize_t write(int fd, const void *buf, size_t count) */
+        p = lock_user(VERIFY_READ, arg1, arg2, 1);
+        if (!p) {
+            ret = -1;
+        } else {
+            ret = write(arg0, p, arg2);
+            unlock_user(p, arg1, 0);
+        }
+        break;
+
+    case TARGET_SYS_close:
+        /* int close(int fd) */
+        /* don't close stdin/stdout/stderr */
+        if (arg0 > 2) {
+            ret = close(arg0);
+        } else {
+            ret = 0;
+        }
+        break;
+
+    case TARGET_SYS_lseek:
+        /* off_t lseek(int fd, off_t offset, int whence */
+        ret = lseek(arg0, arg1, arg2);
+        break;
+
+    case TARGET_SYS_stat:
+        /* int stat(const char *path, struct stat *buf) */
+        p = lock_user_string(arg0);
+        if (!p) {
+            ret = -1;
+        } else {
+            ret = stat(p, &s);
+            unlock_user(p, arg0, 0);
+            if (translate_stat(env, arg1, &s) == 0) {
+                ret = -1;
+            }
+        }
+        break;
+
+    case TARGET_SYS_fstat:
+        /* int stat(int fd, struct stat *buf) */
+        ret = fstat(arg0, &s);
+        if (ret == 0) {
+            if (translate_stat(env, arg1, &s) == 0) {
+                ret = -1;
+            }
+        }
+        break;
+
+    default:
+        /* unhandled */
+        return false;
+    }
+
+    env->regs[R_R1] = ret;
+    return true;
+}
diff --git a/target-lm32/op_helper.c b/target-lm32/op_helper.c
index 2f36b7b..308742a 100644
--- a/target-lm32/op_helper.c
+++ b/target-lm32/op_helper.c
@@ -1,28 +1,18 @@
 #include <assert.h>
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 
 #include "hw/lm32/lm32_pic.h"
 #include "hw/char/lm32_juart.h"
 
-#include "exec/softmmu_exec.h"
+#include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-#define MMUSUFFIX _mmu
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 void raise_exception(CPULM32State *env, int index)
 {
     CPUState *cs = CPU(lm32_env_get_cpu(env));
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index c8abd1f..a51ade9 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -19,13 +19,13 @@
 
 #include "cpu.h"
 #include "disas/disas.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "tcg-op.h"
 
+#include "exec/cpu_ldst.h"
 #include "hw/lm32/lm32_pic.h"
 
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-gen.h"
 
 #define DISAS_LM32 1
 #if DISAS_LM32
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index 077b653..8be9745 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -21,7 +21,7 @@
 #include "cpu.h"
 #include "exec/gdbstub.h"
 
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #define SIGNBIT (1u << 31)
 
diff --git a/target-m68k/helper.h b/target-m68k/helper.h
index 2b02450..f4e5fdf 100644
--- a/target-m68k/helper.h
+++ b/target-m68k/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_1(bitrev, i32, i32)
 DEF_HELPER_1(ff1, i32, i32)
 DEF_HELPER_2(sats, i32, i32, i32)
@@ -50,5 +48,3 @@
 
 DEF_HELPER_2(flush_flags, void, env, i32)
 DEF_HELPER_2(raise_exception, void, env, i32)
-
-#include "exec/def-helper.h"
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 06302b1..9dd3e74 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -17,7 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 #if defined(CONFIG_USER_ONLY)
 
@@ -34,22 +35,6 @@
 
 extern int semihosting_enabled;
 
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index cd66289..50df4d3 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -22,10 +22,10 @@
 #include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 //#define DEBUG_DISPATCH 1
 
diff --git a/target-microblaze/helper.h b/target-microblaze/helper.h
index 4e51429..bd13826 100644
--- a/target-microblaze/helper.h
+++ b/target-microblaze/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_1(debug, void, env)
 DEF_HELPER_FLAGS_3(carry, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
@@ -37,5 +35,3 @@
 
 DEF_HELPER_2(get, i32, i32, i32)
 DEF_HELPER_3(put, void, i32, i32, i32)
-
-#include "exec/def-helper.h"
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index f8fb7f9..a4c8f04 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -20,23 +20,13 @@
 
 #include <assert.h>
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
+#include "exec/cpu_ldst.h"
 
 #define D(x)
 
 #if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-#define SHIFT 3
-#include "exec/softmmu_template.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
  * NULL, it means that the function was called in C code (i.e. not
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index 782a489..c422bdc 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -21,11 +21,10 @@
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "microblaze-decode.h"
-
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/cpu_ldst.h"
+#include "exec/helper-gen.h"
 
 #define SIM_COMPAT 0
 #define DISAS_GNU 1
diff --git a/target-mips/cpu-qom.h b/target-mips/cpu-qom.h
index 8877f81..2cff15a 100644
--- a/target-mips/cpu-qom.h
+++ b/target-mips/cpu-qom.h
@@ -80,5 +80,7 @@
 hwaddr mips_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int mips_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int mips_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void mips_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                  int is_write, int is_user, uintptr_t retaddr);
 
 #endif
diff --git a/target-mips/cpu.c b/target-mips/cpu.c
index ae37ae2..dd954fc 100644
--- a/target-mips/cpu.c
+++ b/target-mips/cpu.c
@@ -137,6 +137,7 @@
     cc->handle_mmu_fault = mips_cpu_handle_mmu_fault;
 #else
     cc->do_unassigned_access = mips_cpu_unassigned_access;
+    cc->do_unaligned_access = mips_cpu_do_unaligned_access;
     cc->get_phys_page_debug = mips_cpu_get_phys_page_debug;
 #endif
 
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index 6c2014e..a9b2c7a 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -3,6 +3,7 @@
 
 //#define DEBUG_OP
 
+#define ALIGNED_ONLY
 #define TARGET_HAS_ICE 1
 
 #define ELF_MACHINE	EM_MIPS
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
index a2f46d9..94083fb 100644
--- a/target-mips/dsp_helper.c
+++ b/target-mips/dsp_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/bitops.h"
 
 /* As the byte ordering doesn't matter, i.e. all columns are treated
diff --git a/target-mips/helper.h b/target-mips/helper.h
index 8c7921a..74ef094 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
 DEF_HELPER_2(raise_exception, noreturn, env, i32)
 
@@ -691,7 +689,3 @@
 #endif
 DEF_HELPER_FLAGS_3(wrdsp, 0, void, tl, tl, env)
 DEF_HELPER_FLAGS_2(rddsp, 0, tl, tl, env)
-
-
-
-#include "exec/def-helper.h"
diff --git a/target-mips/lmi_helper.c b/target-mips/lmi_helper.c
index 1b24353..bbfcd59 100644
--- a/target-mips/lmi_helper.c
+++ b/target-mips/lmi_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /* If the byte ordering doesn't matter, i.e. all columns are treated
    identically, then this union can be used directly.  If byte ordering
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index 4edec6c..4704216 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -19,12 +19,8 @@
 #include <stdlib.h>
 #include "cpu.h"
 #include "qemu/host-utils.h"
-
-#include "helper.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
 static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
@@ -2128,28 +2124,12 @@
 
 #if !defined(CONFIG_USER_ONLY)
 
-static void QEMU_NORETURN do_unaligned_access(CPUMIPSState *env,
-                                              target_ulong addr, int is_write,
-                                              int is_user, uintptr_t retaddr);
-
-#define MMUSUFFIX _mmu
-#define ALIGNED_ONLY
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
-static void do_unaligned_access(CPUMIPSState *env, target_ulong addr,
-                                int is_write, int is_user, uintptr_t retaddr)
+void mips_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
+                                  int is_write, int is_user, uintptr_t retaddr)
 {
+    MIPSCPU *cpu = MIPS_CPU(cs);
+    CPUMIPSState *env = &cpu->env;
+
     env->CP0_BadVAddr = addr;
     do_raise_exception(env, (is_write == 1) ? EXCP_AdES : EXCP_AdEL, retaddr);
 }
diff --git a/target-mips/translate.c b/target-mips/translate.c
index 05f82d2..76deb7b 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -24,10 +24,10 @@
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #define MIPS_DEBUG_DISAS 0
 //#define MIPS_DEBUG_SIGN_EXTENSIONS
diff --git a/target-moxie/helper.c b/target-moxie/helper.c
index 3d0c34d..6c98965 100644
--- a/target-moxie/helper.c
+++ b/target-moxie/helper.c
@@ -25,23 +25,9 @@
 #include "cpu.h"
 #include "mmu.h"
 #include "exec/exec-all.h"
-#include "exec/softmmu_exec.h"
+#include "exec/cpu_ldst.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
+#include "exec/helper-proto.h"
 
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
diff --git a/target-moxie/helper.h b/target-moxie/helper.h
index 3aa5549..d94ef7a 100644
--- a/target-moxie/helper.h
+++ b/target-moxie/helper.h
@@ -1,9 +1,5 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(raise_exception, void, env, int)
 DEF_HELPER_1(debug, void, env)
 
 DEF_HELPER_FLAGS_3(div, TCG_CALL_NO_WG, i32, env, i32, i32)
 DEF_HELPER_FLAGS_3(udiv, TCG_CALL_NO_WG, i32, env, i32, i32)
-
-#include "exec/def-helper.h"
diff --git a/target-moxie/translate.c b/target-moxie/translate.c
index 63f889f..4541b9b 100644
--- a/target-moxie/translate.c
+++ b/target-moxie/translate.c
@@ -32,10 +32,10 @@
 #include "exec/exec-all.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
diff --git a/target-openrisc/exception_helper.c b/target-openrisc/exception_helper.c
index 0c53b77..6093953 100644
--- a/target-openrisc/exception_helper.c
+++ b/target-openrisc/exception_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "exception.h"
 
 void HELPER(exception)(CPUOpenRISCState *env, uint32_t excp)
diff --git a/target-openrisc/fpu_helper.c b/target-openrisc/fpu_helper.c
index 4615a36..c94ed35 100644
--- a/target-openrisc/fpu_helper.c
+++ b/target-openrisc/fpu_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "exception.h"
 
 static inline uint32_t ieee_ex_to_openrisc(OpenRISCCPU *cpu, int fexcp)
diff --git a/target-openrisc/helper.h b/target-openrisc/helper.h
index 2af9790..f53fa21 100644
--- a/target-openrisc/helper.h
+++ b/target-openrisc/helper.h
@@ -17,8 +17,6 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "exec/def-helper.h"
-
 /* exception */
 DEF_HELPER_FLAGS_2(exception, 0, void, env, i32)
 
@@ -66,5 +64,3 @@
 /* sys */
 DEF_HELPER_FLAGS_4(mtspr, 0, void, env, tl, tl, tl)
 DEF_HELPER_FLAGS_4(mfspr, 0, tl, env, tl, tl, tl)
-
-#include "exec/def-helper.h"
diff --git a/target-openrisc/int_helper.c b/target-openrisc/int_helper.c
index 16cb5ab..6e27aeb 100644
--- a/target-openrisc/int_helper.c
+++ b/target-openrisc/int_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "exception.h"
 #include "qemu/host-utils.h"
 
diff --git a/target-openrisc/interrupt_helper.c b/target-openrisc/interrupt_helper.c
index 8194057..55a780c 100644
--- a/target-openrisc/interrupt_helper.c
+++ b/target-openrisc/interrupt_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 void HELPER(rfe)(CPUOpenRISCState *env)
 {
diff --git a/target-openrisc/mmu_helper.c b/target-openrisc/mmu_helper.c
index fb457c7..ee1c6f6 100644
--- a/target-openrisc/mmu_helper.c
+++ b/target-openrisc/mmu_helper.c
@@ -19,22 +19,9 @@
  */
 
 #include "cpu.h"
+#include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
-#include "exec/softmmu_exec.h"
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUState *cs, target_ulong addr, int is_write,
               int mmu_idx, uintptr_t retaddr)
diff --git a/target-openrisc/sys_helper.c b/target-openrisc/sys_helper.c
index fedcbed..53ca6bc 100644
--- a/target-openrisc/sys_helper.c
+++ b/target-openrisc/sys_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #define TO_SPR(group, number) (((group) << 11) + (number))
 
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 852b5e6..b728718 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -26,10 +26,10 @@
 #include "qemu/log.h"
 #include "config.h"
 #include "qemu/bitops.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #define OPENRISC_DISAS
 
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index 4fa297d..7dfc52d 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -17,7 +17,8 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 #include "helper_regs.h"
 
diff --git a/target-ppc/fpu_helper.c b/target-ppc/fpu_helper.c
index c6f484f..cd8f015 100644
--- a/target-ppc/fpu_helper.c
+++ b/target-ppc/fpu_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /*****************************************************************************/
 /* Floating point operations helpers */
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index 99f10de..08f3916 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_3(raise_exception_err, void, env, i32, i32)
 DEF_HELPER_2(raise_exception, void, env, i32)
 DEF_HELPER_4(tw, void, env, tl, tl, i32)
@@ -613,5 +611,3 @@
 DEF_HELPER_3(store_601_batl, void, env, i32, tl)
 DEF_HELPER_3(store_601_batu, void, env, i32, tl)
 #endif
-
-#include "exec/def-helper.h"
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index 18b54f0..588f6a9 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -18,7 +18,7 @@
  */
 #include "cpu.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #include "helper_regs.h"
 /*****************************************************************************/
diff --git a/target-ppc/mem_helper.c b/target-ppc/mem_helper.c
index f35ed03..02b627e 100644
--- a/target-ppc/mem_helper.c
+++ b/target-ppc/mem_helper.c
@@ -18,13 +18,10 @@
  */
 #include "cpu.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #include "helper_regs.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif /* !defined(CONFIG_USER_ONLY) */
+#include "exec/cpu_ldst.h"
 
 //#define DEBUG_OP
 
diff --git a/target-ppc/misc_helper.c b/target-ppc/misc_helper.c
index 2eb2fa6..7331b1b 100644
--- a/target-ppc/misc_helper.c
+++ b/target-ppc/misc_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #include "helper_regs.h"
 
diff --git a/target-ppc/mmu-hash32.c b/target-ppc/mmu-hash32.c
index 1cc1916..0a13a81 100644
--- a/target-ppc/mmu-hash32.c
+++ b/target-ppc/mmu-hash32.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash32.h"
diff --git a/target-ppc/mmu-hash64.c b/target-ppc/mmu-hash64.c
index 1fefe58..c72198a 100644
--- a/target-ppc/mmu-hash64.c
+++ b/target-ppc/mmu-hash64.c
@@ -18,7 +18,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash64.h"
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index 1771863..f029f41 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -17,11 +17,12 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "sysemu/kvm.h"
 #include "kvm_ppc.h"
 #include "mmu-hash64.h"
 #include "mmu-hash32.h"
+#include "exec/cpu_ldst.h"
 
 //#define DEBUG_MMU
 //#define DEBUG_BATS
@@ -2903,22 +2904,6 @@
 
 /*****************************************************************************/
 
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
diff --git a/target-ppc/timebase_helper.c b/target-ppc/timebase_helper.c
index fad738a..865dcbe 100644
--- a/target-ppc/timebase_helper.c
+++ b/target-ppc/timebase_helper.c
@@ -17,7 +17,7 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /*****************************************************************************/
 /* SPR accesses */
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index e3fcb03..f089014 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -22,10 +22,10 @@
 #include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu/host-utils.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 #define CPU_SINGLE_STEP 0x1
 #define CPU_BRANCH_STEP 0x2
diff --git a/target-s390x/cc_helper.c b/target-s390x/cc_helper.c
index 9e676a5..373eb17 100644
--- a/target-s390x/cc_helper.c
+++ b/target-s390x/cc_helper.c
@@ -19,7 +19,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 
 /* #define DEBUG_HELPER */
diff --git a/target-s390x/cpu-qom.h b/target-s390x/cpu-qom.h
index ac0460e..f9c96d1 100644
--- a/target-s390x/cpu-qom.h
+++ b/target-s390x/cpu-qom.h
@@ -86,6 +86,7 @@
 int s390_cpu_write_elf64_qemunote(WriteCoreDumpFunction f,
                                   CPUState *cpu, void *opaque);
 hwaddr s390_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
+hwaddr s390_cpu_get_phys_addr_debug(CPUState *cpu, vaddr addr);
 int s390_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int s390_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
 
diff --git a/target-s390x/fpu_helper.c b/target-s390x/fpu_helper.c
index 3e9c7b2..b946ec1 100644
--- a/target-s390x/fpu_helper.c
+++ b/target-s390x/fpu_helper.c
@@ -19,11 +19,8 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
-
-#if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-#endif
+#include "exec/cpu_ldst.h"
+#include "exec/helper-proto.h"
 
 /* #define DEBUG_HELPER */
 #ifdef DEBUG_HELPER
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index 7c76fc1..67ab106 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -21,6 +21,7 @@
 #include "cpu.h"
 #include "exec/gdbstub.h"
 #include "qemu/timer.h"
+#include "exec/cpu_ldst.h"
 #ifndef CONFIG_USER_ONLY
 #include "sysemu/sysemu.h"
 #endif
@@ -489,6 +490,18 @@
     return raddr;
 }
 
+hwaddr s390_cpu_get_phys_addr_debug(CPUState *cs, vaddr vaddr)
+{
+    hwaddr phys_addr;
+    target_ulong page;
+
+    page = vaddr & TARGET_PAGE_MASK;
+    phys_addr = cpu_get_phys_page_debug(cs, page);
+    phys_addr += (vaddr & ~TARGET_PAGE_MASK);
+
+    return phys_addr;
+}
+
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr)
 {
     if (mask & PSW_MASK_WAIT) {
diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index 0d80aa0..faebfd9 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(exception, noreturn, env, i32)
 DEF_HELPER_FLAGS_4(nc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
 DEF_HELPER_FLAGS_4(oc, TCG_CALL_NO_WG, i32, env, i32, i64, i64)
@@ -115,5 +113,3 @@
 DEF_HELPER_2(lra, i64, env, i64)
 DEF_HELPER_FLAGS_3(stura, TCG_CALL_NO_WG, void, env, i64, i64)
 #endif
-
-#include "exec/def-helper.h"
diff --git a/target-s390x/int_helper.c b/target-s390x/int_helper.c
index 6a929ca..cb8dd98 100644
--- a/target-s390x/int_helper.c
+++ b/target-s390x/int_helper.c
@@ -20,7 +20,7 @@
 
 #include "cpu.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /* #define DEBUG_HELPER */
 #ifdef DEBUG_HELPER
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 56179af..7a07f9d 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -36,6 +36,7 @@
 #include "sysemu/device_tree.h"
 #include "qapi/qmp/qjson.h"
 #include "monitor/monitor.h"
+#include "exec/gdbstub.h"
 #include "trace.h"
 
 /* #define DEBUG_KVM */
@@ -86,6 +87,14 @@
 #define ICPT_CPU_STOP                   0x28
 #define ICPT_IO                         0x40
 
+static CPUWatchpoint hw_watchpoint;
+/*
+ * We don't use a list because this structure is also used to transmit the
+ * hardware breakpoints to the kernel.
+ */
+static struct kvm_hw_breakpoint *hw_breakpoints;
+static int nb_hw_breakpoints;
+
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
@@ -320,12 +329,16 @@
     return mem == MAP_FAILED ? NULL : mem;
 }
 
+/* DIAG 501 is used for sw breakpoints */
+static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
+
 int kvm_arch_insert_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
-    static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
 
-    if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) ||
-        cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)diag_501, 4, 1)) {
+    if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn,
+                            sizeof(diag_501), 0) ||
+        cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)diag_501,
+                            sizeof(diag_501), 1)) {
         return -EINVAL;
     }
     return 0;
@@ -333,38 +346,140 @@
 
 int kvm_arch_remove_sw_breakpoint(CPUState *cs, struct kvm_sw_breakpoint *bp)
 {
-    uint8_t t[4];
-    static const uint8_t diag_501[] = {0x83, 0x24, 0x05, 0x01};
+    uint8_t t[sizeof(diag_501)];
 
-    if (cpu_memory_rw_debug(cs, bp->pc, t, 4, 0)) {
+    if (cpu_memory_rw_debug(cs, bp->pc, t, sizeof(diag_501), 0)) {
         return -EINVAL;
-    } else if (memcmp(t, diag_501, 4)) {
+    } else if (memcmp(t, diag_501, sizeof(diag_501))) {
         return -EINVAL;
-    } else if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn, 1, 1)) {
+    } else if (cpu_memory_rw_debug(cs, bp->pc, (uint8_t *)&bp->saved_insn,
+                                   sizeof(diag_501), 1)) {
         return -EINVAL;
     }
 
     return 0;
 }
 
+static struct kvm_hw_breakpoint *find_hw_breakpoint(target_ulong addr,
+                                                    int len, int type)
+{
+    int n;
+
+    for (n = 0; n < nb_hw_breakpoints; n++) {
+        if (hw_breakpoints[n].addr == addr && hw_breakpoints[n].type == type &&
+            (hw_breakpoints[n].len == len || len == -1)) {
+            return &hw_breakpoints[n];
+        }
+    }
+
+    return NULL;
+}
+
+static int insert_hw_breakpoint(target_ulong addr, int len, int type)
+{
+    int size;
+
+    if (find_hw_breakpoint(addr, len, type)) {
+        return -EEXIST;
+    }
+
+    size = (nb_hw_breakpoints + 1) * sizeof(struct kvm_hw_breakpoint);
+
+    if (!hw_breakpoints) {
+        nb_hw_breakpoints = 0;
+        hw_breakpoints = (struct kvm_hw_breakpoint *)g_try_malloc(size);
+    } else {
+        hw_breakpoints =
+            (struct kvm_hw_breakpoint *)g_try_realloc(hw_breakpoints, size);
+    }
+
+    if (!hw_breakpoints) {
+        nb_hw_breakpoints = 0;
+        return -ENOMEM;
+    }
+
+    hw_breakpoints[nb_hw_breakpoints].addr = addr;
+    hw_breakpoints[nb_hw_breakpoints].len = len;
+    hw_breakpoints[nb_hw_breakpoints].type = type;
+
+    nb_hw_breakpoints++;
+
+    return 0;
+}
+
 int kvm_arch_insert_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type)
 {
-    return -ENOSYS;
+    switch (type) {
+    case GDB_BREAKPOINT_HW:
+        type = KVM_HW_BP;
+        break;
+    case GDB_WATCHPOINT_WRITE:
+        if (len < 1) {
+            return -EINVAL;
+        }
+        type = KVM_HW_WP_WRITE;
+        break;
+    default:
+        return -ENOSYS;
+    }
+    return insert_hw_breakpoint(addr, len, type);
 }
 
 int kvm_arch_remove_hw_breakpoint(target_ulong addr,
                                   target_ulong len, int type)
 {
-    return -ENOSYS;
+    int size;
+    struct kvm_hw_breakpoint *bp = find_hw_breakpoint(addr, len, type);
+
+    if (bp == NULL) {
+        return -ENOENT;
+    }
+
+    nb_hw_breakpoints--;
+    if (nb_hw_breakpoints > 0) {
+        /*
+         * In order to trim the array, move the last element to the position to
+         * be removed - if necessary.
+         */
+        if (bp != &hw_breakpoints[nb_hw_breakpoints]) {
+            *bp = hw_breakpoints[nb_hw_breakpoints];
+        }
+        size = nb_hw_breakpoints * sizeof(struct kvm_hw_breakpoint);
+        hw_breakpoints =
+             (struct kvm_hw_breakpoint *)g_realloc(hw_breakpoints, size);
+    } else {
+        g_free(hw_breakpoints);
+        hw_breakpoints = NULL;
+    }
+
+    return 0;
 }
 
 void kvm_arch_remove_all_hw_breakpoints(void)
 {
+    nb_hw_breakpoints = 0;
+    g_free(hw_breakpoints);
+    hw_breakpoints = NULL;
 }
 
 void kvm_arch_update_guest_debug(CPUState *cpu, struct kvm_guest_debug *dbg)
 {
+    int i;
+
+    if (nb_hw_breakpoints > 0) {
+        dbg->arch.nr_hw_bp = nb_hw_breakpoints;
+        dbg->arch.hw_bp = hw_breakpoints;
+
+        for (i = 0; i < nb_hw_breakpoints; ++i) {
+            hw_breakpoints[i].phys_addr = s390_cpu_get_phys_addr_debug(cpu,
+                                                       hw_breakpoints[i].addr);
+        }
+        dbg->control |= KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP;
+    } else {
+        dbg->arch.nr_hw_bp = 0;
+        dbg->arch.hw_bp = NULL;
+    }
 }
 
 void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run)
@@ -579,6 +694,22 @@
     handle_diag_308(&cpu->env, r1, r3);
 }
 
+static int handle_sw_breakpoint(S390CPU *cpu, struct kvm_run *run)
+{
+    CPUS390XState *env = &cpu->env;
+    unsigned long pc;
+
+    cpu_synchronize_state(CPU(cpu));
+
+    pc = env->psw.addr - 4;
+    if (kvm_find_sw_breakpoint(CPU(cpu), pc)) {
+        env->psw.addr = pc;
+        return EXCP_DEBUG;
+    }
+
+    return -ENOENT;
+}
+
 #define DIAG_KVM_CODE_MASK 0x000000000000ffff
 
 static int handle_diag(S390CPU *cpu, struct kvm_run *run, uint32_t ipb)
@@ -599,7 +730,7 @@
         r = handle_hypercall(cpu, run);
         break;
     case DIAG_KVM_BREAKPOINT:
-        sleep(10);
+        r = handle_sw_breakpoint(cpu, run);
         break;
     default:
         DPRINTF("KVM: unknown DIAG: 0x%x\n", func_code);
@@ -701,7 +832,7 @@
     return 0;
 }
 
-static void handle_instruction(S390CPU *cpu, struct kvm_run *run)
+static int handle_instruction(S390CPU *cpu, struct kvm_run *run)
 {
     unsigned int ipa0 = (run->s390_sieic.ipa & 0xff00);
     uint8_t ipa1 = run->s390_sieic.ipa & 0x00ff;
@@ -728,8 +859,11 @@
     }
 
     if (r < 0) {
+        r = 0;
         enter_pgmcheck(cpu, 0x0001);
     }
+
+    return r;
 }
 
 static bool is_special_wait_psw(CPUState *cs)
@@ -749,7 +883,7 @@
             (long)cs->kvm_run->psw_addr);
     switch (icpt_code) {
         case ICPT_INSTRUCTION:
-            handle_instruction(cpu, run);
+            r = handle_instruction(cpu, run);
             break;
         case ICPT_WAITPSW:
             /* disabled wait, since enabled wait is handled in kernel */
@@ -830,7 +964,36 @@
 
 static int kvm_arch_handle_debug_exit(S390CPU *cpu)
 {
-    return -ENOSYS;
+    CPUState *cs = CPU(cpu);
+    struct kvm_run *run = cs->kvm_run;
+
+    int ret = 0;
+    struct kvm_debug_exit_arch *arch_info = &run->debug.arch;
+
+    switch (arch_info->type) {
+    case KVM_HW_WP_WRITE:
+        if (find_hw_breakpoint(arch_info->addr, -1, arch_info->type)) {
+            cs->watchpoint_hit = &hw_watchpoint;
+            hw_watchpoint.vaddr = arch_info->addr;
+            hw_watchpoint.flags = BP_MEM_WRITE;
+            ret = EXCP_DEBUG;
+        }
+        break;
+    case KVM_HW_BP:
+        if (find_hw_breakpoint(arch_info->addr, -1, arch_info->type)) {
+            ret = EXCP_DEBUG;
+        }
+        break;
+    case KVM_SINGLESTEP:
+        if (cs->singlestep_enabled) {
+            ret = EXCP_DEBUG;
+        }
+        break;
+    default:
+        ret = -ENOSYS;
+    }
+
+    return ret;
 }
 
 int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run)
@@ -911,6 +1074,16 @@
 
 void kvm_arch_init_irq_routing(KVMState *s)
 {
+    /*
+     * Note that while irqchip capabilities generally imply that cpustates
+     * are handled in-kernel, it is not true for s390 (yet); therefore, we
+     * have to override the common code kvm_halt_in_kernel_allowed setting.
+     */
+    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
+        kvm_irqfds_allowed = true;
+        kvm_gsi_routing_allowed = true;
+        kvm_halt_in_kernel_allowed = false;
+    }
 }
 
 int kvm_s390_assign_subch_ioeventfd(EventNotifier *notifier, uint32_t sch,
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
index d8ca300..5a55de8 100644
--- a/target-s390x/mem_helper.c
+++ b/target-s390x/mem_helper.c
@@ -19,26 +19,12 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 /*****************************************************************************/
 /* Softmmu support */
 #if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
 
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
index cdbbb79..9dae025 100644
--- a/target-s390x/misc_helper.c
+++ b/target-s390x/misc_helper.c
@@ -21,16 +21,16 @@
 #include "cpu.h"
 #include "exec/memory.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include <string.h>
 #include "sysemu/kvm.h"
 #include "qemu/timer.h"
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
 #endif
+#include "exec/cpu_ldst.h"
 
 #if !defined(CONFIG_USER_ONLY)
-#include "exec/softmmu_exec.h"
 #include "sysemu/cpus.h"
 #include "sysemu/sysemu.h"
 #include "hw/s390x/ebcdic.h"
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 81b7e33..8ca4824 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -33,14 +33,14 @@
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "qemu/host-utils.h"
+#include "exec/cpu_ldst.h"
 
 /* global register indexes */
 static TCGv_ptr cpu_env;
 
 #include "exec/gen-icount.h"
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 
 /* Information that (most) every instruction needs to manipulate.  */
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index 7162448..3b5c436 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_1(ldtlb, void, env)
 DEF_HELPER_1(raise_illegal_instruction, noreturn, env)
 DEF_HELPER_1(raise_slot_illegal_instruction, noreturn, env)
@@ -46,5 +44,3 @@
 DEF_HELPER_2(ftrc_DT, i32, env, f64)
 DEF_HELPER_3(fipr, void, env, i32, i32)
 DEF_HELPER_2(ftrv, void, env, i32)
-
-#include "exec/def-helper.h"
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index 720a97b..74a5c4e 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -19,24 +19,10 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 #ifndef CONFIG_USER_ONLY
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
 
 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 2360609..8126818 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -23,10 +23,10 @@
 #include "cpu.h"
 #include "disas/disas.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 typedef struct DisasContext {
     struct TranslationBlock *tb;
diff --git a/target-sparc/cc_helper.c b/target-sparc/cc_helper.c
index 63bab07..35dab73 100644
--- a/target-sparc/cc_helper.c
+++ b/target-sparc/cc_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 static uint32_t compute_all_flags(CPUSPARCState *env)
 {
diff --git a/target-sparc/cpu-qom.h b/target-sparc/cpu-qom.h
index 8e3e0de..477c4d5 100644
--- a/target-sparc/cpu-qom.h
+++ b/target-sparc/cpu-qom.h
@@ -81,5 +81,8 @@
 hwaddr sparc_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int sparc_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int sparc_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cpu,
+                                                 vaddr addr, int is_write,
+                                                 int is_user, uintptr_t retaddr);
 
 #endif
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index d9f37e9..3a0ee50 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -825,6 +825,7 @@
     cc->handle_mmu_fault = sparc_cpu_handle_mmu_fault;
 #else
     cc->do_unassigned_access = sparc_cpu_unassigned_access;
+    cc->do_unaligned_access = sparc_cpu_do_unaligned_access;
     cc->get_phys_page_debug = sparc_cpu_get_phys_page_debug;
 #endif
 
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index f72451d..836f87f 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -5,6 +5,8 @@
 #include "qemu-common.h"
 #include "qemu/bswap.h"
 
+#define ALIGNED_ONLY
+
 #if !defined(TARGET_SPARC64)
 #define TARGET_LONG_BITS 32
 #define TARGET_DPREGS 16
diff --git a/target-sparc/fop_helper.c b/target-sparc/fop_helper.c
index f4b62a5..ee4592e 100644
--- a/target-sparc/fop_helper.c
+++ b/target-sparc/fop_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 #define QT0 (env->qt0)
 #define QT1 (env->qt1)
diff --git a/target-sparc/helper.c b/target-sparc/helper.c
index ae7740b..4850c7c 100644
--- a/target-sparc/helper.c
+++ b/target-sparc/helper.c
@@ -19,7 +19,7 @@
 
 #include "cpu.h"
 #include "qemu/host-utils.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "sysemu/sysemu.h"
 
 void helper_raise_exception(CPUSPARCState *env, int tt)
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index cd8d3fa..1ad23e8 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 #ifndef TARGET_SPARC64
 DEF_HELPER_1(rett, void, env)
 DEF_HELPER_2(wrpsr, void, env, tl)
@@ -175,5 +173,3 @@
 #undef VIS_CMPHELPER
 DEF_HELPER_1(compute_psr, void, env)
 DEF_HELPER_1(compute_C_icc, i32, env)
-
-#include "exec/def-helper.h"
diff --git a/target-sparc/int64_helper.c b/target-sparc/int64_helper.c
index bf24232..b02d22b 100644
--- a/target-sparc/int64_helper.c
+++ b/target-sparc/int64_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "trace.h"
 
 #define DEBUG_PCALL
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index ec14802..03bd9f9 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -18,7 +18,8 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 //#define DEBUG_MMU
 //#define DEBUG_MXCC
@@ -64,27 +65,6 @@
 #define QT0 (env->qt0)
 #define QT1 (env->qt1)
 
-#if !defined(CONFIG_USER_ONLY)
-static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
-                                              target_ulong addr, int is_write,
-                                              int is_user, uintptr_t retaddr);
-#include "exec/softmmu_exec.h"
-#define MMUSUFFIX _mmu
-#define ALIGNED_ONLY
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-#endif
-
 #if defined(TARGET_SPARC64) && !defined(CONFIG_USER_ONLY)
 /* Calculates TSB pointer value for fault page size 8k or 64k */
 static uint64_t ultrasparc_tsb_pointer(uint64_t tsb_register,
@@ -2425,11 +2405,13 @@
 #endif
 
 #if !defined(CONFIG_USER_ONLY)
-static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
-                                              target_ulong addr, int is_write,
-                                              int is_user, uintptr_t retaddr)
+void QEMU_NORETURN sparc_cpu_do_unaligned_access(CPUState *cs,
+                                                 vaddr addr, int is_write,
+                                                 int is_user, uintptr_t retaddr)
 {
-    SPARCCPU *cpu = sparc_env_get_cpu(env);
+    SPARCCPU *cpu = SPARC_CPU(cs);
+    CPUSPARCState *env = &cpu->env;
+
 #ifdef DEBUG_UNALIGNED
     printf("Unaligned access to 0x" TARGET_FMT_lx " from 0x" TARGET_FMT_lx
            "\n", addr, env->pc);
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index 2de1c4a..1ab07a1 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -26,11 +26,11 @@
 
 #include "cpu.h"
 #include "disas/disas.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "tcg-op.h"
+#include "exec/cpu_ldst.h"
 
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-gen.h"
 
 #define DEBUG_DISAS
 
diff --git a/target-sparc/vis_helper.c b/target-sparc/vis_helper.c
index 9d2edb0..383cc8b 100644
--- a/target-sparc/vis_helper.c
+++ b/target-sparc/vis_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /* This function uses non-native bit order */
 #define GET_FIELD(X, FROM, TO)                                  \
diff --git a/target-sparc/win_helper.c b/target-sparc/win_helper.c
index 3e82eb7..f01ae08 100644
--- a/target-sparc/win_helper.c
+++ b/target-sparc/win_helper.c
@@ -18,7 +18,7 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "trace.h"
 
 static inline void memcpy32(target_ulong *dst, const target_ulong *src)
diff --git a/target-unicore32/helper.c b/target-unicore32/helper.c
index 169c85c..e5ebbf4 100644
--- a/target-unicore32/helper.c
+++ b/target-unicore32/helper.c
@@ -11,7 +11,7 @@
 
 #include "cpu.h"
 #include "exec/gdbstub.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
 #ifndef CONFIG_USER_ONLY
 #include "ui/console.h"
diff --git a/target-unicore32/helper.h b/target-unicore32/helper.h
index e85ce6c..9418137 100644
--- a/target-unicore32/helper.h
+++ b/target-unicore32/helper.h
@@ -6,7 +6,6 @@
  * published by the Free Software Foundation, or (at your option) any
  * later version. See the COPYING file in the top-level directory.
  */
-#include "exec/def-helper.h"
 
 #ifndef CONFIG_USER_ONLY
 DEF_HELPER_4(cp0_set, void, env, i32, i32, i32)
@@ -64,5 +63,3 @@
 
 DEF_HELPER_2(ucf64_sf2si, f32, f32, env)
 DEF_HELPER_2(ucf64_df2si, f32, f64, env)
-
-#include "exec/def-helper.h"
diff --git a/target-unicore32/op_helper.c b/target-unicore32/op_helper.c
index 4c6950d..0266dbd 100644
--- a/target-unicore32/op_helper.c
+++ b/target-unicore32/op_helper.c
@@ -9,7 +9,8 @@
  * later version. See the COPYING file in the top-level directory.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/cpu_ldst.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
@@ -241,22 +242,6 @@
 }
 
 #ifndef CONFIG_USER_ONLY
-#include "exec/softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
 void tlb_fill(CPUState *cs, target_ulong addr, int is_write,
               int mmu_idx, uintptr_t retaddr)
 {
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index c2402cf..5a8c7c8 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -18,10 +18,10 @@
 #include "disas/disas.h"
 #include "tcg-op.h"
 #include "qemu/log.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 /* internal defines */
 typedef struct DisasContext {
diff --git a/target-unicore32/ucf64_helper.c b/target-unicore32/ucf64_helper.c
index 34fa2a5..0c7ea26 100644
--- a/target-unicore32/ucf64_helper.c
+++ b/target-unicore32/ucf64_helper.c
@@ -9,7 +9,7 @@
  * See the COPYING file in the top-level directory.
  */
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 
 /*
  * The convention used for UniCore-F64 instructions:
diff --git a/target-xtensa/cpu-qom.h b/target-xtensa/cpu-qom.h
index c6cc2d9..f320486 100644
--- a/target-xtensa/cpu-qom.h
+++ b/target-xtensa/cpu-qom.h
@@ -89,5 +89,7 @@
 hwaddr xtensa_cpu_get_phys_page_debug(CPUState *cpu, vaddr addr);
 int xtensa_cpu_gdb_read_register(CPUState *cpu, uint8_t *buf, int reg);
 int xtensa_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
+void xtensa_cpu_do_unaligned_access(CPUState *cpu, vaddr addr,
+                                    int is_write, int is_user, uintptr_t retaddr);
 
 #endif
diff --git a/target-xtensa/cpu.c b/target-xtensa/cpu.c
index 6251f1c..9d8801b 100644
--- a/target-xtensa/cpu.c
+++ b/target-xtensa/cpu.c
@@ -148,6 +148,7 @@
     cc->gdb_read_register = xtensa_cpu_gdb_read_register;
     cc->gdb_write_register = xtensa_cpu_gdb_write_register;
 #ifndef CONFIG_USER_ONLY
+    cc->do_unaligned_access = xtensa_cpu_do_unaligned_access;
     cc->get_phys_page_debug = xtensa_cpu_get_phys_page_debug;
 #endif
     dc->vmsd = &vmstate_xtensa_cpu;
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index e210bac..d797d26 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -28,6 +28,7 @@
 #ifndef CPU_XTENSA_H
 #define CPU_XTENSA_H
 
+#define ALIGNED_ONLY
 #define TARGET_LONG_BITS 32
 #define ELF_MACHINE EM_XTENSA
 
diff --git a/target-xtensa/helper.h b/target-xtensa/helper.h
index 322b04c..ed3af0b 100644
--- a/target-xtensa/helper.h
+++ b/target-xtensa/helper.h
@@ -1,5 +1,3 @@
-#include "exec/def-helper.h"
-
 DEF_HELPER_2(exception, noreturn, env, i32)
 DEF_HELPER_3(exception_cause, noreturn, env, i32, i32)
 DEF_HELPER_4(exception_cause_vaddr, noreturn, env, i32, i32, i32)
@@ -58,5 +56,3 @@
 DEF_HELPER_4(ult_s, void, env, i32, f32, f32)
 DEF_HELPER_4(ole_s, void, env, i32, f32, f32)
 DEF_HELPER_4(ule_s, void, env, i32, f32, f32)
-
-#include "exec/def-helper.h"
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index b531019..dae1386 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -26,33 +26,17 @@
  */
 
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/host-utils.h"
-#include "exec/softmmu_exec.h"
+#include "exec/cpu_ldst.h"
 #include "exec/address-spaces.h"
+#include "qemu/timer.h"
 
-static void do_unaligned_access(CPUXtensaState *env,
-        target_ulong addr, int is_write, int is_user, uintptr_t retaddr);
-
-#define ALIGNED_ONLY
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "exec/softmmu_template.h"
-
-#define SHIFT 1
-#include "exec/softmmu_template.h"
-
-#define SHIFT 2
-#include "exec/softmmu_template.h"
-
-#define SHIFT 3
-#include "exec/softmmu_template.h"
-
-static void do_unaligned_access(CPUXtensaState *env,
-        target_ulong addr, int is_write, int is_user, uintptr_t retaddr)
+void xtensa_cpu_do_unaligned_access(CPUState *cs,
+        vaddr addr, int is_write, int is_user, uintptr_t retaddr)
 {
-    XtensaCPU *cpu = xtensa_env_get_cpu(env);
+    XtensaCPU *cpu = XTENSA_CPU(cs);
+    CPUXtensaState *env = &cpu->env;
 
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_UNALIGNED_EXCEPTION) &&
             !xtensa_option_enabled(env->config, XTENSA_OPTION_HW_ALIGNMENT)) {
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 764cee9..2f22cce 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -36,10 +36,10 @@
 #include "tcg-op.h"
 #include "qemu/log.h"
 #include "sysemu/sysemu.h"
+#include "exec/cpu_ldst.h"
 
-#include "helper.h"
-#define GEN_HELPER 1
-#include "helper.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 typedef struct DisasContext {
     const XtensaConfig *config;
@@ -419,7 +419,7 @@
 static void gen_jumpi(DisasContext *dc, uint32_t dest, int slot)
 {
     TCGv_i32 tmp = tcg_const_i32(dest);
-    if (((dc->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
+    if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
         slot = -1;
     }
     gen_jump_slot(dc, tmp, slot);
@@ -447,7 +447,7 @@
 static void gen_callwi(DisasContext *dc, int callinc, uint32_t dest, int slot)
 {
     TCGv_i32 tmp = tcg_const_i32(dest);
-    if (((dc->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
+    if (((dc->tb->pc ^ dest) & TARGET_PAGE_MASK) != 0) {
         slot = -1;
     }
     gen_callw_slot(dc, callinc, tmp, slot);
diff --git a/target-xtensa/xtensa-semi.c b/target-xtensa/xtensa-semi.c
index 424253d..16e9d8c 100644
--- a/target-xtensa/xtensa-semi.c
+++ b/target-xtensa/xtensa-semi.c
@@ -30,7 +30,7 @@
 #include <string.h>
 #include <stddef.h>
 #include "cpu.h"
-#include "helper.h"
+#include "exec/helper-proto.h"
 #include "qemu/log.h"
 
 enum {
diff --git a/tcg-runtime.c b/tcg-runtime.c
index 4b66e51..9daba69 100644
--- a/tcg-runtime.c
+++ b/tcg-runtime.c
@@ -23,75 +23,85 @@
  */
 #include <stdint.h>
 #include "qemu/host-utils.h"
-#include "tcg/tcg-runtime.h"
+
+/* This file is compiled once, and thus we can't include the standard
+   "exec/helper-proto.h", which has includes that are target specific.  */
+
+#include "exec/helper-head.h"
+
+#define DEF_HELPER_FLAGS_2(name, flags, ret, t1, t2) \
+  dh_ctype(ret) HELPER(name) (dh_ctype(t1), dh_ctype(t2));
+
+#include "tcg-runtime.h"
+
 
 /* 32-bit helpers */
 
-int32_t tcg_helper_div_i32(int32_t arg1, int32_t arg2)
+int32_t HELPER(div_i32)(int32_t arg1, int32_t arg2)
 {
     return arg1 / arg2;
 }
 
-int32_t tcg_helper_rem_i32(int32_t arg1, int32_t arg2)
+int32_t HELPER(rem_i32)(int32_t arg1, int32_t arg2)
 {
     return arg1 % arg2;
 }
 
-uint32_t tcg_helper_divu_i32(uint32_t arg1, uint32_t arg2)
+uint32_t HELPER(divu_i32)(uint32_t arg1, uint32_t arg2)
 {
     return arg1 / arg2;
 }
 
-uint32_t tcg_helper_remu_i32(uint32_t arg1, uint32_t arg2)
+uint32_t HELPER(remu_i32)(uint32_t arg1, uint32_t arg2)
 {
     return arg1 % arg2;
 }
 
 /* 64-bit helpers */
 
-int64_t tcg_helper_shl_i64(int64_t arg1, int64_t arg2)
+uint64_t HELPER(shl_i64)(uint64_t arg1, uint64_t arg2)
 {
     return arg1 << arg2;
 }
 
-int64_t tcg_helper_shr_i64(int64_t arg1, int64_t arg2)
-{
-    return (uint64_t)arg1 >> arg2;
-}
-
-int64_t tcg_helper_sar_i64(int64_t arg1, int64_t arg2)
+uint64_t HELPER(shr_i64)(uint64_t arg1, uint64_t arg2)
 {
     return arg1 >> arg2;
 }
 
-int64_t tcg_helper_div_i64(int64_t arg1, int64_t arg2)
+int64_t HELPER(sar_i64)(int64_t arg1, int64_t arg2)
+{
+    return arg1 >> arg2;
+}
+
+int64_t HELPER(div_i64)(int64_t arg1, int64_t arg2)
 {
     return arg1 / arg2;
 }
 
-int64_t tcg_helper_rem_i64(int64_t arg1, int64_t arg2)
+int64_t HELPER(rem_i64)(int64_t arg1, int64_t arg2)
 {
     return arg1 % arg2;
 }
 
-uint64_t tcg_helper_divu_i64(uint64_t arg1, uint64_t arg2)
+uint64_t HELPER(divu_i64)(uint64_t arg1, uint64_t arg2)
 {
     return arg1 / arg2;
 }
 
-uint64_t tcg_helper_remu_i64(uint64_t arg1, uint64_t arg2)
+uint64_t HELPER(remu_i64)(uint64_t arg1, uint64_t arg2)
 {
     return arg1 % arg2;
 }
 
-uint64_t tcg_helper_muluh_i64(uint64_t arg1, uint64_t arg2)
+uint64_t HELPER(muluh_i64)(uint64_t arg1, uint64_t arg2)
 {
     uint64_t l, h;
     mulu64(&l, &h, arg1, arg2);
     return h;
 }
 
-int64_t tcg_helper_mulsh_i64(int64_t arg1, int64_t arg2)
+int64_t HELPER(mulsh_i64)(int64_t arg1, int64_t arg2)
 {
     uint64_t l, h;
     muls64(&l, &h, arg1, arg2);
diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index 77bb6d9..56dae66 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -1809,24 +1809,23 @@
 }
 
 typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
+    DebugFrameHeader h;
     uint8_t fde_def_cfa[4];
     uint8_t fde_reg_ofs[24];
 } DebugFrame;
 
 #define ELF_HOST_MACHINE EM_AARCH64
 
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 0x78,             /* sleb128 -8 */
-    .cie.return_column = TCG_REG_LR,
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x78,             /* sleb128 -8 */
+    .h.cie.return_column = TCG_REG_LR,
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
         12, TCG_REG_SP,                 /* DW_CFA_def_cfa sp, ... */
@@ -1851,8 +1850,5 @@
 
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (intptr_t)buf;
-    debug_frame.fde.func_len = buf_size;
-
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h
index a32aea6..60c7493 100644
--- a/tcg/aarch64/tcg-target.h
+++ b/tcg/aarch64/tcg-target.h
@@ -99,8 +99,6 @@
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
 {
     __builtin___clear_cache((char *)start, (char *)stop);
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index 538ca2a..e40301c 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -2077,8 +2077,7 @@
 }
 
 typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
+    DebugFrameHeader h;
     uint8_t fde_def_cfa[4];
     uint8_t fde_reg_ofs[18];
 } DebugFrame;
@@ -2088,16 +2087,16 @@
 /* We're expecting a 2 byte uleb128 encoded value.  */
 QEMU_BUILD_BUG_ON(FRAME_SIZE >= (1 << 14));
 
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 0x7c,             /* sleb128 -4 */
-    .cie.return_column = 14,
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
+    .h.cie.return_column = 14,
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
         12, 13,                         /* DW_CFA_def_cfa sp, ... */
@@ -2120,8 +2119,5 @@
 
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (tcg_target_long) buf;
-    debug_frame.fde.func_len = buf_size;
-
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index 73f10c4..1c719e2 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -86,8 +86,6 @@
 #define TCG_TARGET_HAS_div_i32          use_idiv_instructions
 #define TCG_TARGET_HAS_rem_i32          0
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 extern bool tcg_target_deposit_valid(int ofs, int len);
 #define TCG_TARGET_deposit_i32_valid  tcg_target_deposit_valid
 
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index a373073..4133dcf 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -1407,7 +1407,8 @@
         } else {
             retaddr = TCG_REG_RAX;
             tcg_out_movi(s, TCG_TYPE_PTR, retaddr, (uintptr_t)l->raddr);
-            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP, 0);
+            tcg_out_st(s, TCG_TYPE_PTR, retaddr, TCG_REG_ESP,
+                       TCG_TARGET_CALL_STACK_OFFSET);
         }
     }
 
@@ -2341,8 +2342,7 @@
 }
 
 typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
+    DebugFrameHeader h;
     uint8_t fde_def_cfa[4];
     uint8_t fde_reg_ofs[14];
 } DebugFrame;
@@ -2354,16 +2354,16 @@
     /* Host machine without ELF. */
 #elif TCG_TARGET_REG_BITS == 64
 #define ELF_HOST_MACHINE EM_X86_64
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 0x78,             /* sleb128 -8 */
-    .cie.return_column = 16,
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x78,             /* sleb128 -8 */
+    .h.cie.return_column = 16,
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
         12, 7,                          /* DW_CFA_def_cfa %rsp, ... */
@@ -2383,16 +2383,16 @@
 };
 #else
 #define ELF_HOST_MACHINE EM_386
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 0x7c,             /* sleb128 -4 */
-    .cie.return_column = 8,
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 0x7c,             /* sleb128 -4 */
+    .h.cie.return_column = 8,
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
         12, 4,                          /* DW_CFA_def_cfa %esp, ... */
@@ -2413,9 +2413,6 @@
 #if defined(ELF_HOST_MACHINE)
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (uintptr_t)buf;
-    debug_frame.fde.func_len = buf_size;
-
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
 #endif
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index 6c94e5c..7a9980e 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -130,8 +130,6 @@
 #define TCG_TARGET_HAS_mulsh_i64        0
 #endif
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 #define TCG_TARGET_deposit_i32_valid(ofs, len) \
     (((ofs) == 0 && (len) == 8) || ((ofs) == 8 && (len) == 8) || \
      ((ofs) == 0 && (len) == 16))
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index 3a59b50..d675589 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -160,8 +160,6 @@
 #define TCG_TARGET_HAS_mulsh_i64        0
 #define TCG_TARGET_HAS_trunc_shr_i32    0
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 #define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
 #define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
 
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 0ae495c..8855d50 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -24,14 +24,17 @@
  * THE SOFTWARE.
  */
 
-#include "tcg-be-null.h"
+#include "tcg-be-ldst.h"
 
-#if defined(HOST_WORDS_BIGENDIAN) == defined(TARGET_WORDS_BIGENDIAN)
-# define TCG_NEED_BSWAP 0
+#ifdef HOST_WORDS_BIGENDIAN
+# define MIPS_BE  1
 #else
-# define TCG_NEED_BSWAP 1
+# define MIPS_BE  0
 #endif
 
+#define LO_OFF    (MIPS_BE * 4)
+#define HI_OFF    (4 - LO_OFF)
+
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
     "zero",
@@ -64,13 +67,17 @@
     "k1",
     "gp",
     "sp",
-    "fp",
+    "s8",
     "ra",
 };
 #endif
 
+#define TCG_TMP0  TCG_REG_AT
+#define TCG_TMP1  TCG_REG_T9
+
 /* check if we really need so many registers :P */
 static const TCGReg tcg_target_reg_alloc_order[] = {
+    /* Call saved registers.  */
     TCG_REG_S0,
     TCG_REG_S1,
     TCG_REG_S2,
@@ -79,6 +86,10 @@
     TCG_REG_S5,
     TCG_REG_S6,
     TCG_REG_S7,
+    TCG_REG_S8,
+
+    /* Call clobbered registers.  */
+    TCG_REG_T0,
     TCG_REG_T1,
     TCG_REG_T2,
     TCG_REG_T3,
@@ -88,12 +99,14 @@
     TCG_REG_T7,
     TCG_REG_T8,
     TCG_REG_T9,
-    TCG_REG_A0,
-    TCG_REG_A1,
-    TCG_REG_A2,
-    TCG_REG_A3,
+    TCG_REG_V1,
     TCG_REG_V0,
-    TCG_REG_V1
+
+    /* Argument registers, opposite order of allocation.  */
+    TCG_REG_A3,
+    TCG_REG_A2,
+    TCG_REG_A1,
+    TCG_REG_A0,
 };
 
 static const TCGReg tcg_target_call_iarg_regs[4] = {
@@ -142,6 +155,17 @@
     reloc_pc16(code_ptr, (tcg_insn_unit *)value);
 }
 
+#define TCG_CT_CONST_ZERO 0x100
+#define TCG_CT_CONST_U16  0x200    /* Unsigned 16-bit: 0 - 0xffff.  */
+#define TCG_CT_CONST_S16  0x400    /* Signed 16-bit: -32768 - 32767 */
+#define TCG_CT_CONST_P2M1 0x800    /* Power of 2 minus 1.  */
+#define TCG_CT_CONST_N16  0x1000   /* "Negatable" 16-bit: -32767 - 32767 */
+
+static inline bool is_p2m1(tcg_target_long val)
+{
+    return val && ((val + 1) & val) == 0;
+}
+
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -161,11 +185,11 @@
     case 'l': /* qemu_ld input arg constraint */
         ct->ct |= TCG_CT_REG;
         tcg_regset_set(ct->u.regs, 0xffffffff);
-#if defined(CONFIG_SOFTMMU)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
-# if (TARGET_LONG_BITS == 64)
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-# endif
+#if defined(CONFIG_SOFTMMU)
+        if (TARGET_LONG_BITS == 64) {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
+        }
 #endif
         break;
     case 'S': /* qemu_st constraint */
@@ -173,13 +197,12 @@
         tcg_regset_set(ct->u.regs, 0xffffffff);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
 #if defined(CONFIG_SOFTMMU)
-# if (TARGET_LONG_BITS == 32)
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
-# endif
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-# if TARGET_LONG_BITS == 64
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
-# endif
+        if (TARGET_LONG_BITS == 32) {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
+        } else {
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
+            tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
+        }
 #endif
         break;
     case 'I':
@@ -188,6 +211,12 @@
     case 'J':
         ct->ct |= TCG_CT_CONST_S16;
         break;
+    case 'K':
+        ct->ct |= TCG_CT_CONST_P2M1;
+        break;
+    case 'N':
+        ct->ct |= TCG_CT_CONST_N16;
+        break;
     case 'Z':
         /* We are cheating a bit here, using the fact that the register
            ZERO is also the register number 0. Hence there is no need
@@ -208,20 +237,27 @@
 {
     int ct;
     ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST)
+    if (ct & TCG_CT_CONST) {
         return 1;
-    else if ((ct & TCG_CT_CONST_ZERO) && val == 0)
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
         return 1;
-    else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val)
+    } else if ((ct & TCG_CT_CONST_U16) && val == (uint16_t)val) {
         return 1;
-    else if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val)
+    } else if ((ct & TCG_CT_CONST_S16) && val == (int16_t)val) {
         return 1;
-    else
-        return 0;
+    } else if ((ct & TCG_CT_CONST_N16) && val >= -32767 && val <= 32767) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_P2M1)
+               && use_mips32r2_instructions && is_p2m1(val)) {
+        return 1;
+    }
+    return 0;
 }
 
 /* instruction opcodes */
-enum {
+typedef enum {
+    OPC_J        = 0x02 << 26,
+    OPC_JAL      = 0x03 << 26,
     OPC_BEQ      = 0x04 << 26,
     OPC_BNE      = 0x05 << 26,
     OPC_BLEZ     = 0x06 << 26,
@@ -279,16 +315,17 @@
     OPC_MUL      = OPC_SPECIAL2 | 0x002,
 
     OPC_SPECIAL3 = 0x1f << 26,
+    OPC_EXT      = OPC_SPECIAL3 | 0x000,
     OPC_INS      = OPC_SPECIAL3 | 0x004,
     OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
     OPC_SEB      = OPC_SPECIAL3 | 0x420,
     OPC_SEH      = OPC_SPECIAL3 | 0x620,
-};
+} MIPSInsn;
 
 /*
  * Type reg
  */
-static inline void tcg_out_opc_reg(TCGContext *s, int opc,
+static inline void tcg_out_opc_reg(TCGContext *s, MIPSInsn opc,
                                    TCGReg rd, TCGReg rs, TCGReg rt)
 {
     int32_t inst;
@@ -303,7 +340,7 @@
 /*
  * Type immediate
  */
-static inline void tcg_out_opc_imm(TCGContext *s, int opc,
+static inline void tcg_out_opc_imm(TCGContext *s, MIPSInsn opc,
                                    TCGReg rt, TCGReg rs, TCGArg imm)
 {
     int32_t inst;
@@ -316,9 +353,25 @@
 }
 
 /*
+ * Type bitfield
+ */
+static inline void tcg_out_opc_bf(TCGContext *s, MIPSInsn opc, TCGReg rt,
+                                  TCGReg rs, int msb, int lsb)
+{
+    int32_t inst;
+
+    inst = opc;
+    inst |= (rs & 0x1F) << 21;
+    inst |= (rt & 0x1F) << 16;
+    inst |= (msb & 0x1F) << 11;
+    inst |= (lsb & 0x1F) << 6;
+    tcg_out32(s, inst);
+}
+
+/*
  * Type branch
  */
-static inline void tcg_out_opc_br(TCGContext *s, int opc,
+static inline void tcg_out_opc_br(TCGContext *s, MIPSInsn opc,
                                   TCGReg rt, TCGReg rs)
 {
     /* We pay attention here to not modify the branch target by reading
@@ -332,7 +385,7 @@
 /*
  * Type sa
  */
-static inline void tcg_out_opc_sa(TCGContext *s, int opc,
+static inline void tcg_out_opc_sa(TCGContext *s, MIPSInsn opc,
                                   TCGReg rd, TCGReg rt, TCGArg sa)
 {
     int32_t inst;
@@ -345,6 +398,29 @@
 
 }
 
+/*
+ * Type jump.
+ * Returns true if the branch was in range and the insn was emitted.
+ */
+static bool tcg_out_opc_jmp(TCGContext *s, MIPSInsn opc, void *target)
+{
+    uintptr_t dest = (uintptr_t)target;
+    uintptr_t from = (uintptr_t)s->code_ptr + 4;
+    int32_t inst;
+
+    /* The pc-region branch happens within the 256MB region of
+       the delay slot (thus the +4).  */
+    if ((from ^ dest) & -(1 << 28)) {
+        return false;
+    }
+    assert((dest & 3) == 0);
+
+    inst = opc;
+    inst |= (dest >> 2) & 0x3ffffff;
+    tcg_out32(s, inst);
+    return true;
+}
+
 static inline void tcg_out_nop(TCGContext *s)
 {
     tcg_out32(s, 0);
@@ -367,8 +443,10 @@
     } else if (arg == (uint16_t)arg) {
         tcg_out_opc_imm(s, OPC_ORI, reg, TCG_REG_ZERO, arg);
     } else {
-        tcg_out_opc_imm(s, OPC_LUI, reg, 0, arg >> 16);
-        tcg_out_opc_imm(s, OPC_ORI, reg, reg, arg & 0xffff);
+        tcg_out_opc_imm(s, OPC_LUI, reg, TCG_REG_ZERO, arg >> 16);
+        if (arg & 0xffff) {
+            tcg_out_opc_imm(s, OPC_ORI, reg, reg, arg & 0xffff);
+        }
     }
 }
 
@@ -378,14 +456,14 @@
         tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
     } else {
         /* ret and arg can't be register at */
-        if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
+        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
             tcg_abort();
         }
 
-        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
         tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
         tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
     }
 }
 
@@ -396,14 +474,14 @@
         tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
     } else {
         /* ret and arg can't be register at */
-        if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
+        if (ret == TCG_TMP0 || arg == TCG_TMP0) {
             tcg_abort();
         }
 
-        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
         tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
         tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
     }
 }
 
@@ -414,22 +492,22 @@
         tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
     } else {
         /* ret and arg must be different and can't be register at */
-        if (ret == arg || ret == TCG_REG_AT || arg == TCG_REG_AT) {
+        if (ret == arg || ret == TCG_TMP0 || arg == TCG_TMP0) {
             tcg_abort();
         }
 
         tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
 
-        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 24);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 24);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
 
-        tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, arg, 0xff00);
-        tcg_out_opc_sa(s, OPC_SLL, TCG_REG_AT, TCG_REG_AT, 8);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, arg, 0xff00);
+        tcg_out_opc_sa(s, OPC_SLL, TCG_TMP0, TCG_TMP0, 8);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
 
-        tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-        tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff00);
-        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+        tcg_out_opc_sa(s, OPC_SRL, TCG_TMP0, arg, 8);
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP0, TCG_TMP0, 0xff00);
+        tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_TMP0);
     }
 }
 
@@ -453,16 +531,18 @@
     }
 }
 
-static inline void tcg_out_ldst(TCGContext *s, int opc, TCGArg arg,
-                                TCGReg arg1, TCGArg arg2)
+static void tcg_out_ldst(TCGContext *s, MIPSInsn opc, TCGReg data,
+                         TCGReg addr, intptr_t ofs)
 {
-    if (arg2 == (int16_t) arg2) {
-        tcg_out_opc_imm(s, opc, arg, arg1, arg2);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_AT, arg2);
-        tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_AT, TCG_REG_AT, arg1);
-        tcg_out_opc_imm(s, opc, arg, TCG_REG_AT, 0);
+    int16_t lo = ofs;
+    if (ofs != lo) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, ofs - lo);
+        if (addr != TCG_REG_ZERO) {
+            tcg_out_opc_reg(s, OPC_ADDU, TCG_TMP0, TCG_TMP0, addr);
+        }
+        addr = TCG_TMP0;
     }
+    tcg_out_opc_imm(s, opc, data, addr, lo);
 }
 
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
@@ -482,136 +562,137 @@
     if (val == (int16_t)val) {
         tcg_out_opc_imm(s, OPC_ADDIU, reg, reg, val);
     } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_AT, val);
-        tcg_out_opc_reg(s, OPC_ADDU, reg, reg, TCG_REG_AT);
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0, val);
+        tcg_out_opc_reg(s, OPC_ADDU, reg, reg, TCG_TMP0);
     }
 }
 
-/* Helper routines for marshalling helper function arguments into
- * the correct registers and stack.
- * arg_num is where we want to put this argument, and is updated to be ready
- * for the next call. arg is the argument itself. Note that arg_num 0..3 is
- * real registers, 4+ on stack.
- *
- * We provide routines for arguments which are: immediate, 32 bit
- * value in register, 16 and 8 bit values in register (which must be zero
- * extended before use) and 64 bit value in a lo:hi register pair.
- */
-#define DEFINE_TCG_OUT_CALL_IARG(NAME, ARGPARAM)                               \
-    static inline void NAME(TCGContext *s, int *arg_num, ARGPARAM)             \
-    {                                                                          \
-    if (*arg_num < 4) {                                                        \
-        DEFINE_TCG_OUT_CALL_IARG_GET_ARG(tcg_target_call_iarg_regs[*arg_num]); \
-    } else {                                                                   \
-        DEFINE_TCG_OUT_CALL_IARG_GET_ARG(TCG_REG_AT);                          \
-        tcg_out_st(s, TCG_TYPE_I32, TCG_REG_AT, TCG_REG_SP, 4 * (*arg_num));   \
-    }                                                                          \
-    (*arg_num)++;                                                              \
-}
-#define DEFINE_TCG_OUT_CALL_IARG_GET_ARG(A) \
-    tcg_out_opc_imm(s, OPC_ANDI, A, arg, 0xff);
-DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_reg8, TCGReg arg)
-#undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
-#define DEFINE_TCG_OUT_CALL_IARG_GET_ARG(A) \
-    tcg_out_opc_imm(s, OPC_ANDI, A, arg, 0xffff);
-DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_reg16, TCGReg arg)
-#undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
-#define DEFINE_TCG_OUT_CALL_IARG_GET_ARG(A) \
-    tcg_out_movi(s, TCG_TYPE_I32, A, arg);
-DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_imm32, TCGArg arg)
-#undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
+/* Bit 0 set if inversion required; bit 1 set if swapping required.  */
+#define MIPS_CMP_INV  1
+#define MIPS_CMP_SWAP 2
 
-/* We don't use the macro for this one to avoid an unnecessary reg-reg
-   move when storing to the stack. */
-static inline void tcg_out_call_iarg_reg32(TCGContext *s, int *arg_num,
-                                           TCGReg arg)
+static const uint8_t mips_cmp_map[16] = {
+    [TCG_COND_LT]  = 0,
+    [TCG_COND_LTU] = 0,
+    [TCG_COND_GE]  = MIPS_CMP_INV,
+    [TCG_COND_GEU] = MIPS_CMP_INV,
+    [TCG_COND_LE]  = MIPS_CMP_INV | MIPS_CMP_SWAP,
+    [TCG_COND_LEU] = MIPS_CMP_INV | MIPS_CMP_SWAP,
+    [TCG_COND_GT]  = MIPS_CMP_SWAP,
+    [TCG_COND_GTU] = MIPS_CMP_SWAP,
+};
+
+static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGReg arg1, TCGReg arg2)
 {
-    if (*arg_num < 4) {
-        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[*arg_num], arg);
-    } else {
-        tcg_out_st(s, TCG_TYPE_I32, arg, TCG_REG_SP, 4 * (*arg_num));
-    }
-    (*arg_num)++;
-}
-
-static inline void tcg_out_call_iarg_reg64(TCGContext *s, int *arg_num,
-                                           TCGReg arg_low, TCGReg arg_high)
-{
-    (*arg_num) = (*arg_num + 1) & ~1;
-
-#if defined(HOST_WORDS_BIGENDIAN)
-    tcg_out_call_iarg_reg32(s, arg_num, arg_high);
-    tcg_out_call_iarg_reg32(s, arg_num, arg_low);
-#else
-    tcg_out_call_iarg_reg32(s, arg_num, arg_low);
-    tcg_out_call_iarg_reg32(s, arg_num, arg_high);
-#endif
-}
-
-static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGArg arg1,
-                           TCGArg arg2, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
+    MIPSInsn s_opc = OPC_SLTU;
+    int cmp_map;
 
     switch (cond) {
     case TCG_COND_EQ:
-        tcg_out_opc_br(s, OPC_BEQ, arg1, arg2);
+        if (arg2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
+            arg1 = ret;
+        }
+        tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, 1);
+        break;
+
+    case TCG_COND_NE:
+        if (arg2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
+            arg1 = ret;
+        }
+        tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, arg1);
+        break;
+
+    case TCG_COND_LT:
+    case TCG_COND_GE:
+    case TCG_COND_LE:
+    case TCG_COND_GT:
+        s_opc = OPC_SLT;
+        /* FALLTHRU */
+
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+    case TCG_COND_LEU:
+    case TCG_COND_GTU:
+        cmp_map = mips_cmp_map[cond];
+        if (cmp_map & MIPS_CMP_SWAP) {
+            TCGReg t = arg1;
+            arg1 = arg2;
+            arg2 = t;
+        }
+        tcg_out_opc_reg(s, s_opc, ret, arg1, arg2);
+        if (cmp_map & MIPS_CMP_INV) {
+            tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
+        }
+        break;
+
+     default:
+         tcg_abort();
+         break;
+     }
+}
+
+static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGReg arg1,
+                           TCGReg arg2, int label_index)
+{
+    static const MIPSInsn b_zero[16] = {
+        [TCG_COND_LT] = OPC_BLTZ,
+        [TCG_COND_GT] = OPC_BGTZ,
+        [TCG_COND_LE] = OPC_BLEZ,
+        [TCG_COND_GE] = OPC_BGEZ,
+    };
+
+    TCGLabel *l;
+    MIPSInsn s_opc = OPC_SLTU;
+    MIPSInsn b_opc;
+    int cmp_map;
+
+    switch (cond) {
+    case TCG_COND_EQ:
+        b_opc = OPC_BEQ;
         break;
     case TCG_COND_NE:
-        tcg_out_opc_br(s, OPC_BNE, arg1, arg2);
+        b_opc = OPC_BNE;
         break;
+
     case TCG_COND_LT:
-        if (arg2 == 0) {
-            tcg_out_opc_br(s, OPC_BLTZ, 0, arg1);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
-        }
-        break;
-    case TCG_COND_LTU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
-        break;
+    case TCG_COND_GT:
+    case TCG_COND_LE:
     case TCG_COND_GE:
         if (arg2 == 0) {
-            tcg_out_opc_br(s, OPC_BGEZ, 0, arg1);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+            b_opc = b_zero[cond];
+            arg2 = arg1;
+            arg1 = 0;
+            break;
         }
-        break;
-    case TCG_COND_GEU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
-        break;
-    case TCG_COND_LE:
-        if (arg2 == 0) {
-            tcg_out_opc_br(s, OPC_BLEZ, 0, arg1);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
-        }
-        break;
-    case TCG_COND_LEU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
-        break;
-    case TCG_COND_GT:
-        if (arg2 == 0) {
-            tcg_out_opc_br(s, OPC_BGTZ, 0, arg1);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
-        }
-        break;
+        s_opc = OPC_SLT;
+        /* FALLTHRU */
+
+    case TCG_COND_LTU:
     case TCG_COND_GTU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+    case TCG_COND_LEU:
+    case TCG_COND_GEU:
+        cmp_map = mips_cmp_map[cond];
+        if (cmp_map & MIPS_CMP_SWAP) {
+            TCGReg t = arg1;
+            arg1 = arg2;
+            arg2 = t;
+        }
+        tcg_out_opc_reg(s, s_opc, TCG_TMP0, arg1, arg2);
+        b_opc = (cmp_map & MIPS_CMP_INV ? OPC_BEQ : OPC_BNE);
+        arg1 = TCG_TMP0;
+        arg2 = TCG_REG_ZERO;
         break;
+
     default:
         tcg_abort();
         break;
     }
+
+    tcg_out_opc_br(s, b_opc, arg1, arg2);
+    l = &s->labels[label_index];
     if (l->has_value) {
         reloc_pc16(s->code_ptr - 1, l->u.value_ptr);
     } else {
@@ -620,913 +701,869 @@
     tcg_out_nop(s);
 }
 
-/* XXX: we implement it at the target level to avoid having to
-   handle cross basic blocks temporaries */
-static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGArg arg1,
-                            TCGArg arg2, TCGArg arg3, TCGArg arg4,
-                            int label_index)
+static TCGReg tcg_out_reduce_eq2(TCGContext *s, TCGReg tmp0, TCGReg tmp1,
+                                 TCGReg al, TCGReg ah,
+                                 TCGReg bl, TCGReg bh)
 {
-    tcg_insn_unit *label_ptr;
+    /* Merge highpart comparison into AH.  */
+    if (bh != 0) {
+        if (ah != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, tmp0, ah, bh);
+            ah = tmp0;
+        } else {
+            ah = bh;
+        }
+    }
+    /* Merge lowpart comparison into AL.  */
+    if (bl != 0) {
+        if (al != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, tmp1, al, bl);
+            al = tmp1;
+        } else {
+            al = bl;
+        }
+    }
+    /* Merge high and low part comparisons into AL.  */
+    if (ah != 0) {
+        if (al != 0) {
+            tcg_out_opc_reg(s, OPC_OR, tmp0, ah, al);
+            al = tmp0;
+        } else {
+            al = ah;
+        }
+    }
+    return al;
+}
 
-    switch(cond) {
+static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
+                             TCGReg al, TCGReg ah, TCGReg bl, TCGReg bh)
+{
+    TCGReg tmp0 = TCG_TMP0;
+    TCGReg tmp1 = ret;
+
+    assert(ret != TCG_TMP0);
+    if (ret == ah || ret == bh) {
+        assert(ret != TCG_TMP1);
+        tmp1 = TCG_TMP1;
+    }
+
+    switch (cond) {
+    case TCG_COND_EQ:
     case TCG_COND_NE:
-        tcg_out_brcond(s, TCG_COND_NE, arg2, arg4, label_index);
-        tcg_out_brcond(s, TCG_COND_NE, arg1, arg3, label_index);
-        return;
-    case TCG_COND_EQ:
+        tmp1 = tcg_out_reduce_eq2(s, tmp0, tmp1, al, ah, bl, bh);
+        tcg_out_setcond(s, cond, ret, tmp1, TCG_REG_ZERO);
         break;
-    case TCG_COND_LT:
-    case TCG_COND_LE:
-        tcg_out_brcond(s, TCG_COND_LT, arg2, arg4, label_index);
-        break;
-    case TCG_COND_GT:
-    case TCG_COND_GE:
-        tcg_out_brcond(s, TCG_COND_GT, arg2, arg4, label_index);
-        break;
-    case TCG_COND_LTU:
-    case TCG_COND_LEU:
-        tcg_out_brcond(s, TCG_COND_LTU, arg2, arg4, label_index);
-        break;
-    case TCG_COND_GTU:
-    case TCG_COND_GEU:
-        tcg_out_brcond(s, TCG_COND_GTU, arg2, arg4, label_index);
-        break;
+
     default:
-        tcg_abort();
+        tcg_out_setcond(s, TCG_COND_EQ, tmp0, ah, bh);
+        tcg_out_setcond(s, tcg_unsigned_cond(cond), tmp1, al, bl);
+        tcg_out_opc_reg(s, OPC_AND, tmp1, tmp1, tmp0);
+        tcg_out_setcond(s, tcg_high_cond(cond), tmp0, ah, bh);
+        tcg_out_opc_reg(s, OPC_OR, ret, tmp1, tmp0);
+        break;
+    }
+}
+
+static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
+                            TCGReg bl, TCGReg bh, int label_index)
+{
+    TCGCond b_cond = TCG_COND_NE;
+    TCGReg tmp = TCG_TMP1;
+
+    /* With branches, we emit between 4 and 9 insns with 2 or 3 branches.
+       With setcond, we emit between 3 and 10 insns and only 1 branch,
+       which ought to get better branch prediction.  */
+     switch (cond) {
+     case TCG_COND_EQ:
+     case TCG_COND_NE:
+        b_cond = cond;
+        tmp = tcg_out_reduce_eq2(s, TCG_TMP0, TCG_TMP1, al, ah, bl, bh);
+        break;
+
+    default:
+        /* Minimize code size by prefering a compare not requiring INV.  */
+        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
+            cond = tcg_invert_cond(cond);
+            b_cond = TCG_COND_EQ;
+        }
+        tcg_out_setcond2(s, cond, tmp, al, ah, bl, bh);
+        break;
     }
 
-    label_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BNE, arg2, arg4);
-    tcg_out_nop(s);
-
-    switch(cond) {
-    case TCG_COND_EQ:
-        tcg_out_brcond(s, TCG_COND_EQ, arg1, arg3, label_index);
-        break;
-    case TCG_COND_LT:
-    case TCG_COND_LTU:
-        tcg_out_brcond(s, TCG_COND_LTU, arg1, arg3, label_index);
-        break;
-    case TCG_COND_LE:
-    case TCG_COND_LEU:
-        tcg_out_brcond(s, TCG_COND_LEU, arg1, arg3, label_index);
-        break;
-    case TCG_COND_GT:
-    case TCG_COND_GTU:
-        tcg_out_brcond(s, TCG_COND_GTU, arg1, arg3, label_index);
-        break;
-    case TCG_COND_GE:
-    case TCG_COND_GEU:
-        tcg_out_brcond(s, TCG_COND_GEU, arg1, arg3, label_index);
-        break;
-    default:
-        tcg_abort();
-    }
-
-    reloc_pc16(label_ptr, s->code_ptr);
+    tcg_out_brcond(s, b_cond, tmp, TCG_REG_ZERO, label_index);
 }
 
 static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGArg c1, TCGArg c2, TCGArg v)
+                            TCGReg c1, TCGReg c2, TCGReg v)
 {
+    MIPSInsn m_opc = OPC_MOVN;
+
     switch (cond) {
     case TCG_COND_EQ:
-        if (c1 == 0) {
-            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c2);
-        } else if (c2 == 0) {
-            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c1);
-        } else {
-            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
-            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
-        }
-        break;
+        m_opc = OPC_MOVZ;
+        /* FALLTHRU */
     case TCG_COND_NE:
-        if (c1 == 0) {
-            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c2);
-        } else if (c2 == 0) {
-            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c1);
-        } else {
-            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
-            tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        if (c2 != 0) {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_TMP0, c1, c2);
+            c1 = TCG_TMP0;
         }
         break;
-    case TCG_COND_LT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
-        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_LTU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
-        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_GE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
-        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_GEU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
-        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_LE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
-        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_LEU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
-        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_GT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
-        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
-        break;
-    case TCG_COND_GTU:
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
-        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
-        break;
+
     default:
-        tcg_abort();
+        /* Minimize code size by prefering a compare not requiring INV.  */
+        if (mips_cmp_map[cond] & MIPS_CMP_INV) {
+            cond = tcg_invert_cond(cond);
+            m_opc = OPC_MOVZ;
+        }
+        tcg_out_setcond(s, cond, TCG_TMP0, c1, c2);
+        c1 = TCG_TMP0;
         break;
     }
+
+    tcg_out_opc_reg(s, m_opc, ret, v, c1);
+}
+
+static void tcg_out_call_int(TCGContext *s, tcg_insn_unit *arg, bool tail)
+{
+    /* Note that the ABI requires the called function's address to be
+       loaded into T9, even if a direct branch is in range.  */
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (uintptr_t)arg);
+
+    /* But do try a direct branch, allowing the cpu better insn prefetch.  */
+    if (tail) {
+        if (!tcg_out_opc_jmp(s, OPC_J, arg)) {
+            tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_T9, 0);
+        }
+    } else {
+        if (!tcg_out_opc_jmp(s, OPC_JAL, arg)) {
+            tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
+        }
+    }
 }
 
-static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
-                            TCGArg arg1, TCGArg arg2)
+static void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
 {
-    switch (cond) {
-    case TCG_COND_EQ:
-        if (arg1 == 0) {
-            tcg_out_opc_imm(s, OPC_SLTIU, ret, arg2, 1);
-        } else if (arg2 == 0) {
-            tcg_out_opc_imm(s, OPC_SLTIU, ret, arg1, 1);
-        } else {
-            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
-            tcg_out_opc_imm(s, OPC_SLTIU, ret, ret, 1);
-        }
-        break;
-    case TCG_COND_NE:
-        if (arg1 == 0) {
-            tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, arg2);
-        } else if (arg2 == 0) {
-            tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, arg1);
-        } else {
-            tcg_out_opc_reg(s, OPC_XOR, ret, arg1, arg2);
-            tcg_out_opc_reg(s, OPC_SLTU, ret, TCG_REG_ZERO, ret);
-        }
-        break;
-    case TCG_COND_LT:
-        tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2);
-        break;
-    case TCG_COND_LTU:
-        tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2);
-        break;
-    case TCG_COND_GE:
-        tcg_out_opc_reg(s, OPC_SLT, ret, arg1, arg2);
-        tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
-        break;
-    case TCG_COND_GEU:
-        tcg_out_opc_reg(s, OPC_SLTU, ret, arg1, arg2);
-        tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
-        break;
-    case TCG_COND_LE:
-        tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1);
-        tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
-        break;
-    case TCG_COND_LEU:
-        tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1);
-        tcg_out_opc_imm(s, OPC_XORI, ret, ret, 1);
-        break;
-    case TCG_COND_GT:
-        tcg_out_opc_reg(s, OPC_SLT, ret, arg2, arg1);
-        break;
-    case TCG_COND_GTU:
-        tcg_out_opc_reg(s, OPC_SLTU, ret, arg2, arg1);
-        break;
-    default:
-        tcg_abort();
-        break;
-    }
-}
-
-/* XXX: we implement it at the target level to avoid having to
-   handle cross basic blocks temporaries */
-static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
-                             TCGArg arg1, TCGArg arg2, TCGArg arg3, TCGArg arg4)
-{
-    switch (cond) {
-    case TCG_COND_EQ:
-        tcg_out_setcond(s, TCG_COND_EQ, TCG_REG_AT, arg2, arg4);
-        tcg_out_setcond(s, TCG_COND_EQ, TCG_REG_T0, arg1, arg3);
-        tcg_out_opc_reg(s, OPC_AND, ret, TCG_REG_AT, TCG_REG_T0);
-        return;
-    case TCG_COND_NE:
-        tcg_out_setcond(s, TCG_COND_NE, TCG_REG_AT, arg2, arg4);
-        tcg_out_setcond(s, TCG_COND_NE, TCG_REG_T0, arg1, arg3);
-        tcg_out_opc_reg(s, OPC_OR, ret, TCG_REG_AT, TCG_REG_T0);
-        return;
-    case TCG_COND_LT:
-    case TCG_COND_LE:
-        tcg_out_setcond(s, TCG_COND_LT, TCG_REG_AT, arg2, arg4);
-        break;
-    case TCG_COND_GT:
-    case TCG_COND_GE:
-        tcg_out_setcond(s, TCG_COND_GT, TCG_REG_AT, arg2, arg4);
-        break;
-    case TCG_COND_LTU:
-    case TCG_COND_LEU:
-        tcg_out_setcond(s, TCG_COND_LTU, TCG_REG_AT, arg2, arg4);
-        break;
-    case TCG_COND_GTU:
-    case TCG_COND_GEU:
-        tcg_out_setcond(s, TCG_COND_GTU, TCG_REG_AT, arg2, arg4);
-        break;
-    default:
-        tcg_abort();
-        break;
-    }
-
-    tcg_out_setcond(s, TCG_COND_EQ, TCG_REG_T0, arg2, arg4);
-
-    switch(cond) {
-    case TCG_COND_LT:
-    case TCG_COND_LTU:
-        tcg_out_setcond(s, TCG_COND_LTU, ret, arg1, arg3);
-        break;
-    case TCG_COND_LE:
-    case TCG_COND_LEU:
-        tcg_out_setcond(s, TCG_COND_LEU, ret, arg1, arg3);
-        break;
-    case TCG_COND_GT:
-    case TCG_COND_GTU:
-        tcg_out_setcond(s, TCG_COND_GTU, ret, arg1, arg3);
-        break;
-    case TCG_COND_GE:
-    case TCG_COND_GEU:
-        tcg_out_setcond(s, TCG_COND_GEU, ret, arg1, arg3);
-        break;
-    default:
-        tcg_abort();
-    }
-
-    tcg_out_opc_reg(s, OPC_AND, ret, ret, TCG_REG_T0);
-    tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+    tcg_out_call_int(s, arg, false);
+    tcg_out_nop(s);
 }
 
 #if defined(CONFIG_SOFTMMU)
-/* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
-   int mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    helper_ldb_mmu,
-    helper_ldw_mmu,
-    helper_ldl_mmu,
-    helper_ldq_mmu,
+static void * const qemu_ld_helpers[16] = {
+    [MO_UB]   = helper_ret_ldub_mmu,
+    [MO_SB]   = helper_ret_ldsb_mmu,
+    [MO_LEUW] = helper_le_lduw_mmu,
+    [MO_LESW] = helper_le_ldsw_mmu,
+    [MO_LEUL] = helper_le_ldul_mmu,
+    [MO_LEQ]  = helper_le_ldq_mmu,
+    [MO_BEUW] = helper_be_lduw_mmu,
+    [MO_BESW] = helper_be_ldsw_mmu,
+    [MO_BEUL] = helper_be_ldul_mmu,
+    [MO_BEQ]  = helper_be_ldq_mmu,
 };
 
-/* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
-   uintxx_t val, int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    helper_stb_mmu,
-    helper_stw_mmu,
-    helper_stl_mmu,
-    helper_stq_mmu,
+static void * const qemu_st_helpers[16] = {
+    [MO_UB]   = helper_ret_stb_mmu,
+    [MO_LEUW] = helper_le_stw_mmu,
+    [MO_LEUL] = helper_le_stl_mmu,
+    [MO_LEQ]  = helper_le_stq_mmu,
+    [MO_BEUW] = helper_be_stw_mmu,
+    [MO_BEUL] = helper_be_stl_mmu,
+    [MO_BEQ]  = helper_be_stq_mmu,
 };
-#endif
 
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
-                            int opc)
+/* Helper routines for marshalling helper function arguments into
+ * the correct registers and stack.
+ * I is where we want to put this argument, and is updated and returned
+ * for the next call. ARG is the argument itself.
+ *
+ * We provide routines for arguments which are: immediate, 32 bit
+ * value in register, 16 and 8 bit values in register (which must be zero
+ * extended before use) and 64 bit value in a lo:hi register pair.
+ */
+
+static int tcg_out_call_iarg_reg(TCGContext *s, int i, TCGReg arg)
 {
-    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
-#if defined(CONFIG_SOFTMMU)
-    tcg_insn_unit *label1_ptr, *label2_ptr;
-    int arg_num;
-    int mem_index, s_bits;
-    int addr_meml;
-# if TARGET_LONG_BITS == 64
-    tcg_insn_unit *label3_ptr;
-    TCGReg addr_regh;
-    int addr_memh;
-# endif
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[i], arg);
+    } else {
+        tcg_out_st(s, TCG_TYPE_REG, arg, TCG_REG_SP, 4 * i);
+    }
+    return i + 1;
+}
+
+static int tcg_out_call_iarg_reg8(TCGContext *s, int i, TCGReg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tmp = tcg_target_call_iarg_regs[i];
+    }
+    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xff);
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_reg16(TCGContext *s, int i, TCGReg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+        tmp = tcg_target_call_iarg_regs[i];
+    }
+    tcg_out_opc_imm(s, OPC_ANDI, tmp, arg, 0xffff);
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_imm(TCGContext *s, int i, TCGArg arg)
+{
+    TCGReg tmp = TCG_TMP0;
+    if (arg == 0) {
+        tmp = TCG_REG_ZERO;
+    } else {
+        if (i < ARRAY_SIZE(tcg_target_call_iarg_regs)) {
+            tmp = tcg_target_call_iarg_regs[i];
+        }
+        tcg_out_movi(s, TCG_TYPE_REG, tmp, arg);
+    }
+    return tcg_out_call_iarg_reg(s, i, tmp);
+}
+
+static int tcg_out_call_iarg_reg2(TCGContext *s, int i, TCGReg al, TCGReg ah)
+{
+    i = (i + 1) & ~1;
+    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? ah : al));
+    i = tcg_out_call_iarg_reg(s, i, (MIPS_BE ? al : ah));
+    return i;
+}
+
+/* Perform the tlb comparison operation.  The complete host address is
+   placed in BASE.  Clobbers AT, T0, A0.  */
+static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl,
+                             TCGReg addrh, int mem_index, TCGMemOp s_bits,
+                             tcg_insn_unit *label_ptr[2], bool is_load)
+{
+    int cmp_off
+        = (is_load
+           ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
+           : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write));
+    int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend);
+
+    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addrl,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
+    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0,
+                    (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
+    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, TCG_AREG0);
+
+    /* Compensate for very large offsets.  */
+    if (add_off >= 0x8000) {
+        /* Most target env are smaller than 32k; none are larger than 64k.
+           Simplify the logic here merely to offset by 0x7ff0, giving us a
+           range just shy of 64k.  Check this assumption.  */
+        QEMU_BUILD_BUG_ON(offsetof(CPUArchState,
+                                   tlb_table[NB_MMU_MODES - 1][1])
+                          > 0x7ff0 + 0x7fff);
+        tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_A0, TCG_REG_A0, 0x7ff0);
+        cmp_off -= 0x7ff0;
+        add_off -= 0x7ff0;
+    }
+
+    /* Load the tlb comparator.  */
+    tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + LO_OFF);
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_opc_imm(s, OPC_LW, base, TCG_REG_A0, cmp_off + HI_OFF);
+    }
+
+    /* Mask the page bits, keeping the alignment bits to compare against.
+       In between, load the tlb addend for the fast path.  */
+    tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1,
+                 TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+    tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off);
+    tcg_out_opc_reg(s, OPC_AND, TCG_TMP1, TCG_TMP1, addrl);
+
+    label_ptr[0] = s->code_ptr;
+    tcg_out_opc_br(s, OPC_BNE, TCG_TMP1, TCG_TMP0);
+
+    if (TARGET_LONG_BITS == 64) {
+        /* delay slot */
+        tcg_out_nop(s);
+
+        label_ptr[1] = s->code_ptr;
+        tcg_out_opc_br(s, OPC_BNE, addrh, base);
+    }
+
+    /* delay slot */
+    tcg_out_opc_reg(s, OPC_ADDU, base, TCG_REG_A0, addrl);
+}
+
+static void add_qemu_ldst_label(TCGContext *s, int is_ld, TCGMemOp opc,
+                                TCGReg datalo, TCGReg datahi,
+                                TCGReg addrlo, TCGReg addrhi,
+                                int mem_index, void *raddr,
+                                tcg_insn_unit *label_ptr[2])
+{
+    TCGLabelQemuLdst *label = new_ldst_label(s);
+
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = datalo;
+    label->datahi_reg = datahi;
+    label->addrlo_reg = addrlo;
+    label->addrhi_reg = addrhi;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS == 64) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOp opc = l->opc;
+    TCGReg v0;
+    int i;
+
+    /* resolve label address */
+    reloc_pc16(l->label_ptr[0], s->code_ptr);
+    if (TARGET_LONG_BITS == 64) {
+        reloc_pc16(l->label_ptr[1], s->code_ptr);
+    }
+
+    i = 1;
+    if (TARGET_LONG_BITS == 64) {
+        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
+    } else {
+        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
+    }
+    i = tcg_out_call_iarg_imm(s, i, l->mem_index);
+    i = tcg_out_call_iarg_imm(s, i, (intptr_t)l->raddr);
+    tcg_out_call_int(s, qemu_ld_helpers[opc], false);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+
+    v0 = l->datalo_reg;
+    if ((opc & MO_SIZE) == MO_64) {
+        /* We eliminated V0 from the possible output registers, so it
+           cannot be clobbered here.  So we must move V1 first.  */
+        if (MIPS_BE) {
+            tcg_out_mov(s, TCG_TYPE_I32, v0, TCG_REG_V1);
+            v0 = l->datahi_reg;
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, l->datahi_reg, TCG_REG_V1);
+        }
+    }
+
+    reloc_pc16(s->code_ptr, l->raddr);
+    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_REG, v0, TCG_REG_V0);
+}
+
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
+{
+    TCGMemOp opc = l->opc;
+    TCGMemOp s_bits = opc & MO_SIZE;
+    int i;
+
+    /* resolve label address */
+    reloc_pc16(l->label_ptr[0], s->code_ptr);
+    if (TARGET_LONG_BITS == 64) {
+        reloc_pc16(l->label_ptr[1], s->code_ptr);
+    }
+
+    i = 1;
+    if (TARGET_LONG_BITS == 64) {
+        i = tcg_out_call_iarg_reg2(s, i, l->addrlo_reg, l->addrhi_reg);
+    } else {
+        i = tcg_out_call_iarg_reg(s, i, l->addrlo_reg);
+    }
+    switch (s_bits) {
+    case MO_8:
+        i = tcg_out_call_iarg_reg8(s, i, l->datalo_reg);
+        break;
+    case MO_16:
+        i = tcg_out_call_iarg_reg16(s, i, l->datalo_reg);
+        break;
+    case MO_32:
+        i = tcg_out_call_iarg_reg(s, i, l->datalo_reg);
+        break;
+    case MO_64:
+        i = tcg_out_call_iarg_reg2(s, i, l->datalo_reg, l->datahi_reg);
+        break;
+    default:
+        tcg_abort();
+    }
+    i = tcg_out_call_iarg_imm(s, i, l->mem_index);
+
+    /* Tail call to the store helper.  Thus force the return address
+       computation to take place in the return address register.  */
+    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_RA, (intptr_t)l->raddr);
+    i = tcg_out_call_iarg_reg(s, i, TCG_REG_RA);
+    tcg_out_call_int(s, qemu_st_helpers[opc], true);
+    /* delay slot */
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0], TCG_AREG0);
+}
 #endif
+
+static void tcg_out_qemu_ld_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, TCGMemOp opc)
+{
+    switch (opc) {
+    case MO_UB:
+        tcg_out_opc_imm(s, OPC_LBU, datalo, base, 0);
+        break;
+    case MO_SB:
+        tcg_out_opc_imm(s, OPC_LB, datalo, base, 0);
+        break;
+    case MO_UW | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
+        tcg_out_bswap16(s, datalo, TCG_TMP1);
+        break;
+    case MO_UW:
+        tcg_out_opc_imm(s, OPC_LHU, datalo, base, 0);
+        break;
+    case MO_SW | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LHU, TCG_TMP1, base, 0);
+        tcg_out_bswap16s(s, datalo, TCG_TMP1);
+        break;
+    case MO_SW:
+        tcg_out_opc_imm(s, OPC_LH, datalo, base, 0);
+        break;
+    case MO_UL | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, 0);
+        tcg_out_bswap32(s, datalo, TCG_TMP1);
+        break;
+    case MO_UL:
+        tcg_out_opc_imm(s, OPC_LW, datalo, base, 0);
+        break;
+    case MO_Q | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, HI_OFF);
+        tcg_out_bswap32(s, datalo, TCG_TMP1);
+        tcg_out_opc_imm(s, OPC_LW, TCG_TMP1, base, LO_OFF);
+        tcg_out_bswap32(s, datahi, TCG_TMP1);
+        break;
+    case MO_Q:
+        tcg_out_opc_imm(s, OPC_LW, datalo, base, LO_OFF);
+        tcg_out_opc_imm(s, OPC_LW, datahi, base, HI_OFF);
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg addr_regl, addr_regh __attribute__((unused));
+    TCGReg data_regl, data_regh;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    tcg_insn_unit *label_ptr[2];
+    int mem_index;
+    TCGMemOp s_bits;
+#endif
+    /* Note that we've eliminated V0 from the output registers,
+       so we won't overwrite the base register during loading.  */
+    TCGReg base = TCG_REG_V0;
+
     data_regl = *args++;
-    if (opc == 3)
-        data_regh = *args++;
-    else
-        data_regh = 0;
+    data_regh = (is_64 ? *args++ : 0);
     addr_regl = *args++;
+    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+
 #if defined(CONFIG_SOFTMMU)
-# if TARGET_LONG_BITS == 64
-    addr_regh = *args++;
-#  if defined(HOST_WORDS_BIGENDIAN)
-    addr_memh = 0;
-    addr_meml = 4;
-#  else
-    addr_memh = 4;
-    addr_meml = 0;
-#  endif
-# else
-    addr_meml = 0;
-# endif
+    mem_index = *args;
+    s_bits = opc & MO_SIZE;
+
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
+                     s_bits, label_ptr, 1);
+    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
+    add_qemu_ldst_label(s, 1, opc, data_regl, data_regh, addr_regl, addr_regh,
+                        mem_index, s->code_ptr, label_ptr);
+#else
+    if (GUEST_BASE == 0 && data_regl != addr_regl) {
+        base = addr_regl;
+    } else if (GUEST_BASE == (int16_t)GUEST_BASE) {
+        tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+        tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
+    }
+    tcg_out_qemu_ld_direct(s, data_regl, data_regh, base, opc);
+#endif
+}
+
+static void tcg_out_qemu_st_direct(TCGContext *s, TCGReg datalo, TCGReg datahi,
+                                   TCGReg base, TCGMemOp opc)
+{
+    switch (opc) {
+    case MO_8:
+        tcg_out_opc_imm(s, OPC_SB, datalo, base, 0);
+        break;
+
+    case MO_16 | MO_BSWAP:
+        tcg_out_opc_imm(s, OPC_ANDI, TCG_TMP1, datalo, 0xffff);
+        tcg_out_bswap16(s, TCG_TMP1, TCG_TMP1);
+        datalo = TCG_TMP1;
+        /* FALLTHRU */
+    case MO_16:
+        tcg_out_opc_imm(s, OPC_SH, datalo, base, 0);
+        break;
+
+    case MO_32 | MO_BSWAP:
+        tcg_out_bswap32(s, TCG_TMP1, datalo);
+        datalo = TCG_TMP1;
+        /* FALLTHRU */
+    case MO_32:
+        tcg_out_opc_imm(s, OPC_SW, datalo, base, 0);
+        break;
+
+    case MO_64 | MO_BSWAP:
+        tcg_out_bswap32(s, TCG_TMP1, datalo);
+        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, HI_OFF);
+        tcg_out_bswap32(s, TCG_TMP1, datahi);
+        tcg_out_opc_imm(s, OPC_SW, TCG_TMP1, base, LO_OFF);
+        break;
+    case MO_64:
+        tcg_out_opc_imm(s, OPC_SW, datalo, base, LO_OFF);
+        tcg_out_opc_imm(s, OPC_SW, datahi, base, HI_OFF);
+        break;
+
+    default:
+        tcg_abort();
+    }
+}
+
+static void tcg_out_addsub2(TCGContext *s, TCGReg rl, TCGReg rh, TCGReg al,
+                            TCGReg ah, TCGArg bl, TCGArg bh, bool cbl,
+                            bool cbh, bool is_sub)
+{
+    TCGReg th = TCG_TMP1;
+
+    /* If we have a negative constant such that negating it would
+       make the high part zero, we can (usually) eliminate one insn.  */
+    if (cbl && cbh && bh == -1 && bl != 0) {
+        bl = -bl;
+        bh = 0;
+        is_sub = !is_sub;
+    }
+
+    /* By operating on the high part first, we get to use the final
+       carry operation to move back from the temporary.  */
+    if (!cbh) {
+        tcg_out_opc_reg(s, (is_sub ? OPC_SUBU : OPC_ADDU), th, ah, bh);
+    } else if (bh != 0 || ah == rl) {
+        tcg_out_opc_imm(s, OPC_ADDIU, th, ah, (is_sub ? -bh : bh));
+    } else {
+        th = ah;
+    }
+
+    /* Note that tcg optimization should eliminate the bl == 0 case.  */
+    if (is_sub) {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, al, bl);
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, -bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, al, bl);
+            tcg_out_opc_reg(s, OPC_SUBU, rl, al, bl);
+        }
+        tcg_out_opc_reg(s, OPC_SUBU, rh, th, TCG_TMP0);
+    } else {
+        if (cbl) {
+            tcg_out_opc_imm(s, OPC_ADDIU, rl, al, bl);
+            tcg_out_opc_imm(s, OPC_SLTIU, TCG_TMP0, rl, bl);
+        } else {
+            tcg_out_opc_reg(s, OPC_ADDU, rl, al, bl);
+            tcg_out_opc_reg(s, OPC_SLTU, TCG_TMP0, rl, (rl == bl ? al : bl));
+        }
+        tcg_out_opc_reg(s, OPC_ADDU, rh, th, TCG_TMP0);
+    }
+}
+
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is_64)
+{
+    TCGReg addr_regl, addr_regh __attribute__((unused));
+    TCGReg data_regl, data_regh, base;
+    TCGMemOp opc;
+#if defined(CONFIG_SOFTMMU)
+    tcg_insn_unit *label_ptr[2];
+    int mem_index;
+    TCGMemOp s_bits;
+#endif
+
+    data_regl = *args++;
+    data_regh = (is_64 ? *args++ : 0);
+    addr_regl = *args++;
+    addr_regh = (TARGET_LONG_BITS == 64 ? *args++ : 0);
+    opc = *args++;
+
+#if defined(CONFIG_SOFTMMU)
     mem_index = *args;
     s_bits = opc & 3;
-#endif
 
-    if (opc == 3) {
-#if defined(HOST_WORDS_BIGENDIAN)
-        data_reg1 = data_regh;
-        data_reg2 = data_regl;
+    /* Note that we eliminated the helper's address argument,
+       so we can reuse that for the base.  */
+    base = (TARGET_LONG_BITS == 32 ? TCG_REG_A1 : TCG_REG_A2);
+    tcg_out_tlb_load(s, base, addr_regl, addr_regh, mem_index,
+                     s_bits, label_ptr, 1);
+    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
+    add_qemu_ldst_label(s, 0, opc, data_regl, data_regh, addr_regl, addr_regh,
+                        mem_index, s->code_ptr, label_ptr);
 #else
-        data_reg1 = data_regl;
-        data_reg2 = data_regh;
-#endif
+    if (GUEST_BASE == 0) {
+        base = addr_regl;
     } else {
-        data_reg1 = data_regl;
-        data_reg2 = 0;
-    }
-#if defined(CONFIG_SOFTMMU)
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, TCG_AREG0);
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_AT, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addr_read) + addr_meml);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T0, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-    tcg_out_opc_reg(s, OPC_AND, TCG_REG_T0, TCG_REG_T0, addr_regl);
-
-# if TARGET_LONG_BITS == 64
-    label3_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BNE, TCG_REG_T0, TCG_REG_AT);
-    tcg_out_nop(s);
-
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_AT, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addr_read) + addr_memh);
-
-    label1_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, addr_regh, TCG_REG_AT);
-    tcg_out_nop(s);
-
-    reloc_pc16(label3_ptr, s->code_ptr);
-# else
-    label1_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_T0, TCG_REG_AT);
-    tcg_out_nop(s);
-# endif
-
-    /* slow path */
-    arg_num = 0;
-    tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# if TARGET_LONG_BITS == 64
-    tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
-# else
-    tcg_out_call_iarg_reg32(s, &arg_num, addr_regl);
-# endif
-    tcg_out_call_iarg_imm32(s, &arg_num, mem_index);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T9, (tcg_target_long)qemu_ld_helpers[s_bits]);
-    tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
-    tcg_out_nop(s);
-
-    switch(opc) {
-    case 0:
-        tcg_out_opc_imm(s, OPC_ANDI, data_reg1, TCG_REG_V0, 0xff);
-        break;
-    case 0 | 4:
-        tcg_out_ext8s(s, data_reg1, TCG_REG_V0);
-        break;
-    case 1:
-        tcg_out_opc_imm(s, OPC_ANDI, data_reg1, TCG_REG_V0, 0xffff);
-        break;
-    case 1 | 4:
-        tcg_out_ext16s(s, data_reg1, TCG_REG_V0);
-        break;
-    case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg1, TCG_REG_V0);
-        break;
-    case 3:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_V1);
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg1, TCG_REG_V0);
-        break;
-    default:
-        tcg_abort();
-    }
-
-    label2_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
-    tcg_out_nop(s);
-
-    /* label1: fast path */
-    reloc_pc16(label1_ptr, s->code_ptr);
-
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addend));
-    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_V0, TCG_REG_A0, addr_regl);
-#else
-    if (GUEST_BASE == (int16_t)GUEST_BASE) {
-        tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_V0, addr_regl, GUEST_BASE);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, GUEST_BASE);
-        tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_V0, TCG_REG_V0, addr_regl);
-    }
-#endif
-
-    switch(opc) {
-    case 0:
-        tcg_out_opc_imm(s, OPC_LBU, data_reg1, TCG_REG_V0, 0);
-        break;
-    case 0 | 4:
-        tcg_out_opc_imm(s, OPC_LB, data_reg1, TCG_REG_V0, 0);
-        break;
-    case 1:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_opc_imm(s, OPC_LHU, TCG_REG_T0, TCG_REG_V0, 0);
-            tcg_out_bswap16(s, data_reg1, TCG_REG_T0);
+        base = TCG_REG_A0;
+        if (GUEST_BASE == (int16_t)GUEST_BASE) {
+            tcg_out_opc_imm(s, OPC_ADDIU, base, addr_regl, GUEST_BASE);
         } else {
-            tcg_out_opc_imm(s, OPC_LHU, data_reg1, TCG_REG_V0, 0);
+            tcg_out_movi(s, TCG_TYPE_PTR, base, GUEST_BASE);
+            tcg_out_opc_reg(s, OPC_ADDU, base, base, addr_regl);
         }
-        break;
-    case 1 | 4:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_opc_imm(s, OPC_LHU, TCG_REG_T0, TCG_REG_V0, 0);
-            tcg_out_bswap16s(s, data_reg1, TCG_REG_T0);
-        } else {
-            tcg_out_opc_imm(s, OPC_LH, data_reg1, TCG_REG_V0, 0);
-        }
-        break;
-    case 2:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_opc_imm(s, OPC_LW, TCG_REG_T0, TCG_REG_V0, 0);
-            tcg_out_bswap32(s, data_reg1, TCG_REG_T0);
-        } else {
-            tcg_out_opc_imm(s, OPC_LW, data_reg1, TCG_REG_V0, 0);
-        }
-        break;
-    case 3:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_opc_imm(s, OPC_LW, TCG_REG_T0, TCG_REG_V0, 4);
-            tcg_out_bswap32(s, data_reg1, TCG_REG_T0);
-            tcg_out_opc_imm(s, OPC_LW, TCG_REG_T0, TCG_REG_V0, 0);
-            tcg_out_bswap32(s, data_reg2, TCG_REG_T0);
-        } else {
-            tcg_out_opc_imm(s, OPC_LW, data_reg1, TCG_REG_V0, 0);
-            tcg_out_opc_imm(s, OPC_LW, data_reg2, TCG_REG_V0, 4);
-        }
-        break;
-    default:
-        tcg_abort();
     }
-
-#if defined(CONFIG_SOFTMMU)
-    reloc_pc16(label2_ptr, s->code_ptr);
+    tcg_out_qemu_st_direct(s, data_regl, data_regh, base, opc);
 #endif
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
-                            int opc)
-{
-    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
-#if defined(CONFIG_SOFTMMU)
-    tcg_insn_unit *label1_ptr, *label2_ptr;
-    int arg_num;
-    int mem_index, s_bits;
-    int addr_meml;
-#endif
-#if TARGET_LONG_BITS == 64
-# if defined(CONFIG_SOFTMMU)
-    tcg_insn_unit *label3_ptr;
-    TCGReg addr_regh;
-    int addr_memh;
-# endif
-#endif
-    data_regl = *args++;
-    if (opc == 3) {
-        data_regh = *args++;
-    } else {
-        data_regh = 0;
-    }
-    addr_regl = *args++;
-#if defined(CONFIG_SOFTMMU)
-# if TARGET_LONG_BITS == 64
-    addr_regh = *args++;
-#  if defined(HOST_WORDS_BIGENDIAN)
-    addr_memh = 0;
-    addr_meml = 4;
-#  else
-    addr_memh = 4;
-    addr_meml = 0;
-#  endif
-# else
-    addr_meml = 0;
-# endif
-    mem_index = *args;
-    s_bits = opc;
-#endif
-
-    if (opc == 3) {
-#if defined(HOST_WORDS_BIGENDIAN)
-        data_reg1 = data_regh;
-        data_reg2 = data_regl;
-#else
-        data_reg1 = data_regl;
-        data_reg2 = data_regh;
-#endif
-    } else {
-        data_reg1 = data_regl;
-        data_reg2 = 0;
-    }
-
-#if defined(CONFIG_SOFTMMU)
-    tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, TCG_AREG0);
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_AT, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addr_write) + addr_meml);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T0, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
-    tcg_out_opc_reg(s, OPC_AND, TCG_REG_T0, TCG_REG_T0, addr_regl);
-
-# if TARGET_LONG_BITS == 64
-    label3_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BNE, TCG_REG_T0, TCG_REG_AT);
-    tcg_out_nop(s);
-
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_AT, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addr_write) + addr_memh);
-
-    label1_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, addr_regh, TCG_REG_AT);
-    tcg_out_nop(s);
-
-    reloc_pc16(label3_ptr, s->code_ptr);
-# else
-    label1_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_T0, TCG_REG_AT);
-    tcg_out_nop(s);
-# endif
-
-    /* slow path */
-    arg_num = 0;
-    tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# if TARGET_LONG_BITS == 64
-    tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
-# else
-    tcg_out_call_iarg_reg32(s, &arg_num, addr_regl);
-# endif
-    switch(opc) {
-    case 0:
-        tcg_out_call_iarg_reg8(s, &arg_num, data_regl);
-        break;
-    case 1:
-        tcg_out_call_iarg_reg16(s, &arg_num, data_regl);
-        break;
-    case 2:
-        tcg_out_call_iarg_reg32(s, &arg_num, data_regl);
-        break;
-    case 3:
-        tcg_out_call_iarg_reg64(s, &arg_num, data_regl, data_regh);
-        break;
-    default:
-        tcg_abort();
-    }
-    tcg_out_call_iarg_imm32(s, &arg_num, mem_index);
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T9, (tcg_target_long)qemu_st_helpers[s_bits]);
-    tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
-    tcg_out_nop(s);
-
-    label2_ptr = s->code_ptr;
-    tcg_out_opc_br(s, OPC_BEQ, TCG_REG_ZERO, TCG_REG_ZERO);
-    tcg_out_nop(s);
-
-    /* label1: fast path */
-    reloc_pc16(label1_ptr, s->code_ptr);
-
-    tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0,
-                    offsetof(CPUArchState, tlb_table[mem_index][0].addend));
-    tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, addr_regl);
-#else
-    if (GUEST_BASE == (int16_t)GUEST_BASE) {
-        tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_A0, addr_regl, GUEST_BASE);
-    } else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A0, GUEST_BASE);
-        tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_A0, TCG_REG_A0, addr_regl);
-    }
-
-#endif
-
-    switch(opc) {
-    case 0:
-        tcg_out_opc_imm(s, OPC_SB, data_reg1, TCG_REG_A0, 0);
-        break;
-    case 1:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_T0, data_reg1, 0xffff);
-            tcg_out_bswap16(s, TCG_REG_T0, TCG_REG_T0);
-            tcg_out_opc_imm(s, OPC_SH, TCG_REG_T0, TCG_REG_A0, 0);
-        } else {
-            tcg_out_opc_imm(s, OPC_SH, data_reg1, TCG_REG_A0, 0);
-        }
-        break;
-    case 2:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_bswap32(s, TCG_REG_T0, data_reg1);
-            tcg_out_opc_imm(s, OPC_SW, TCG_REG_T0, TCG_REG_A0, 0);
-        } else {
-            tcg_out_opc_imm(s, OPC_SW, data_reg1, TCG_REG_A0, 0);
-        }
-        break;
-    case 3:
-        if (TCG_NEED_BSWAP) {
-            tcg_out_bswap32(s, TCG_REG_T0, data_reg2);
-            tcg_out_opc_imm(s, OPC_SW, TCG_REG_T0, TCG_REG_A0, 0);
-            tcg_out_bswap32(s, TCG_REG_T0, data_reg1);
-            tcg_out_opc_imm(s, OPC_SW, TCG_REG_T0, TCG_REG_A0, 4);
-        } else {
-            tcg_out_opc_imm(s, OPC_SW, data_reg1, TCG_REG_A0, 0);
-            tcg_out_opc_imm(s, OPC_SW, data_reg2, TCG_REG_A0, 4);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-
-#if defined(CONFIG_SOFTMMU)
-    reloc_pc16(label2_ptr, s->code_ptr);
-#endif
-}
-
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target)
-{
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T9, (intptr_t)target);
-    tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, TCG_REG_T9, 0);
-    tcg_out_nop(s);
-}
-
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg *args, const int *const_args)
 {
-    switch(opc) {
+    MIPSInsn i1, i2;
+    TCGArg a0, a1, a2;
+    int c2;
+
+    a0 = args[0];
+    a1 = args[1];
+    a2 = args[2];
+    c2 = const_args[2];
+
+    switch (opc) {
     case INDEX_op_exit_tb:
-        tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_V0, args[0]);
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_AT, (uintptr_t)tb_ret_addr);
-        tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_AT, 0);
-        tcg_out_nop(s);
+        {
+            TCGReg b0 = TCG_REG_ZERO;
+
+            if (a0 & ~0xffff) {
+                tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_V0, a0 & ~0xffff);
+                b0 = TCG_REG_V0;
+            }
+            if (!tcg_out_opc_jmp(s, OPC_J, tb_ret_addr)) {
+                tcg_out_movi(s, TCG_TYPE_PTR, TCG_TMP0,
+                             (uintptr_t)tb_ret_addr);
+                tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
+            }
+            tcg_out_opc_imm(s, OPC_ORI, TCG_REG_V0, b0, a0 & 0xffff);
+        }
         break;
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
             /* direct jump method */
-            tcg_abort();
+            s->tb_jmp_offset[a0] = tcg_current_code_size(s);
+            /* Avoid clobbering the address during retranslation.  */
+            tcg_out32(s, OPC_J | (*(uint32_t *)s->code_ptr & 0x3ffffff));
         } else {
             /* indirect jump method */
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_AT,
-                         (uintptr_t)(s->tb_next + args[0]));
-            tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_AT, TCG_REG_AT, 0);
-            tcg_out_opc_reg(s, OPC_JR, 0, TCG_REG_AT, 0);
+            tcg_out_ld(s, TCG_TYPE_PTR, TCG_TMP0, TCG_REG_ZERO,
+                       (uintptr_t)(s->tb_next + a0));
+            tcg_out_opc_reg(s, OPC_JR, 0, TCG_TMP0, 0);
         }
         tcg_out_nop(s);
-        s->tb_next_offset[args[0]] = tcg_current_code_size(s);
+        s->tb_next_offset[a0] = tcg_current_code_size(s);
         break;
     case INDEX_op_br:
-        tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, args[0]);
+        tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, a0);
         break;
 
     case INDEX_op_ld8u_i32:
-        tcg_out_ldst(s, OPC_LBU, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_LBU;
+        goto do_ldst;
     case INDEX_op_ld8s_i32:
-        tcg_out_ldst(s, OPC_LB, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_LB;
+        goto do_ldst;
     case INDEX_op_ld16u_i32:
-        tcg_out_ldst(s, OPC_LHU, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_LHU;
+        goto do_ldst;
     case INDEX_op_ld16s_i32:
-        tcg_out_ldst(s, OPC_LH, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_LH;
+        goto do_ldst;
     case INDEX_op_ld_i32:
-        tcg_out_ldst(s, OPC_LW, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_LW;
+        goto do_ldst;
     case INDEX_op_st8_i32:
-        tcg_out_ldst(s, OPC_SB, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_SB;
+        goto do_ldst;
     case INDEX_op_st16_i32:
-        tcg_out_ldst(s, OPC_SH, args[0], args[1], args[2]);
-        break;
+        i1 = OPC_SH;
+        goto do_ldst;
     case INDEX_op_st_i32:
-        tcg_out_ldst(s, OPC_SW, args[0], args[1], args[2]);
+        i1 = OPC_SW;
+    do_ldst:
+        tcg_out_ldst(s, i1, a0, a1, a2);
         break;
 
     case INDEX_op_add_i32:
-        if (const_args[2]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_ADDU, args[0], args[1], args[2]);
+        i1 = OPC_ADDU, i2 = OPC_ADDIU;
+        goto do_binary;
+    case INDEX_op_or_i32:
+        i1 = OPC_OR, i2 = OPC_ORI;
+        goto do_binary;
+    case INDEX_op_xor_i32:
+        i1 = OPC_XOR, i2 = OPC_XORI;
+    do_binary:
+        if (c2) {
+            tcg_out_opc_imm(s, i2, a0, a1, a2);
+            break;
         }
-        break;
-    case INDEX_op_add2_i32:
-        if (const_args[4]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_AT, args[2], args[4]);
-        } else {
-            tcg_out_opc_reg(s, OPC_ADDU, TCG_REG_AT, args[2], args[4]);
-        }
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_T0, TCG_REG_AT, args[2]);
-        if (const_args[5]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, args[1], args[3], args[5]);
-        } else {
-             tcg_out_opc_reg(s, OPC_ADDU, args[1], args[3], args[5]);
-        }
-        tcg_out_opc_reg(s, OPC_ADDU, args[1], args[1], TCG_REG_T0);
-        tcg_out_mov(s, TCG_TYPE_I32, args[0], TCG_REG_AT);
-        break;
-    case INDEX_op_sub_i32:
-        if (const_args[2]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, args[0], args[1], -args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_SUBU, args[0], args[1], args[2]);
-        }
-        break;
-    case INDEX_op_sub2_i32:
-        if (const_args[4]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, TCG_REG_AT, args[2], -args[4]);
-        } else {
-            tcg_out_opc_reg(s, OPC_SUBU, TCG_REG_AT, args[2], args[4]);
-        }
-        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_T0, args[2], TCG_REG_AT);
-        if (const_args[5]) {
-            tcg_out_opc_imm(s, OPC_ADDIU, args[1], args[3], -args[5]);
-        } else {
-             tcg_out_opc_reg(s, OPC_SUBU, args[1], args[3], args[5]);
-        }
-        tcg_out_opc_reg(s, OPC_SUBU, args[1], args[1], TCG_REG_T0);
-        tcg_out_mov(s, TCG_TYPE_I32, args[0], TCG_REG_AT);
-        break;
-    case INDEX_op_mul_i32:
-        if (use_mips32_instructions) {
-            tcg_out_opc_reg(s, OPC_MUL, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
-            tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-        }
-        break;
-    case INDEX_op_muls2_i32:
-        tcg_out_opc_reg(s, OPC_MULT, 0, args[2], args[3]);
-        tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-        tcg_out_opc_reg(s, OPC_MFHI, args[1], 0, 0);
-        break;
-    case INDEX_op_mulu2_i32:
-        tcg_out_opc_reg(s, OPC_MULTU, 0, args[2], args[3]);
-        tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-        tcg_out_opc_reg(s, OPC_MFHI, args[1], 0, 0);
-        break;
-    case INDEX_op_mulsh_i32:
-        tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
-        break;
-    case INDEX_op_muluh_i32:
-        tcg_out_opc_reg(s, OPC_MULTU, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
-        break;
-    case INDEX_op_div_i32:
-        tcg_out_opc_reg(s, OPC_DIV, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-        break;
-    case INDEX_op_divu_i32:
-        tcg_out_opc_reg(s, OPC_DIVU, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
-        break;
-    case INDEX_op_rem_i32:
-        tcg_out_opc_reg(s, OPC_DIV, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
-        break;
-    case INDEX_op_remu_i32:
-        tcg_out_opc_reg(s, OPC_DIVU, 0, args[1], args[2]);
-        tcg_out_opc_reg(s, OPC_MFHI, args[0], 0, 0);
+    do_binaryv:
+        tcg_out_opc_reg(s, i1, a0, a1, a2);
         break;
 
+    case INDEX_op_sub_i32:
+        if (c2) {
+            tcg_out_opc_imm(s, OPC_ADDIU, a0, a1, -a2);
+            break;
+        }
+        i1 = OPC_SUBU;
+        goto do_binary;
     case INDEX_op_and_i32:
-        if (const_args[2]) {
-            tcg_out_opc_imm(s, OPC_ANDI, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_AND, args[0], args[1], args[2]);
+        if (c2 && a2 != (uint16_t)a2) {
+            int msb = ctz32(~a2) - 1;
+            assert(use_mips32r2_instructions);
+            assert(is_p2m1(a2));
+            tcg_out_opc_bf(s, OPC_EXT, a0, a1, msb, 0);
+            break;
         }
-        break;
-    case INDEX_op_or_i32:
-        if (const_args[2]) {
-            tcg_out_opc_imm(s, OPC_ORI, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_OR, args[0], args[1], args[2]);
-        }
-        break;
+        i1 = OPC_AND, i2 = OPC_ANDI;
+        goto do_binary;
     case INDEX_op_nor_i32:
-        tcg_out_opc_reg(s, OPC_NOR, args[0], args[1], args[2]);
-        break;
-    case INDEX_op_not_i32:
-        tcg_out_opc_reg(s, OPC_NOR, args[0], TCG_REG_ZERO, args[1]);
-        break;
-    case INDEX_op_xor_i32:
-        if (const_args[2]) {
-            tcg_out_opc_imm(s, OPC_XORI, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_XOR, args[0], args[1], args[2]);
+        i1 = OPC_NOR;
+        goto do_binaryv;
+
+    case INDEX_op_mul_i32:
+        if (use_mips32_instructions) {
+            tcg_out_opc_reg(s, OPC_MUL, a0, a1, a2);
+            break;
         }
+        i1 = OPC_MULT, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_mulsh_i32:
+        i1 = OPC_MULT, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_muluh_i32:
+        i1 = OPC_MULTU, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_div_i32:
+        i1 = OPC_DIV, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_divu_i32:
+        i1 = OPC_DIVU, i2 = OPC_MFLO;
+        goto do_hilo1;
+    case INDEX_op_rem_i32:
+        i1 = OPC_DIV, i2 = OPC_MFHI;
+        goto do_hilo1;
+    case INDEX_op_remu_i32:
+        i1 = OPC_DIVU, i2 = OPC_MFHI;
+    do_hilo1:
+        tcg_out_opc_reg(s, i1, 0, a1, a2);
+        tcg_out_opc_reg(s, i2, a0, 0, 0);
+        break;
+
+    case INDEX_op_muls2_i32:
+        i1 = OPC_MULT;
+        goto do_hilo2;
+    case INDEX_op_mulu2_i32:
+        i1 = OPC_MULTU;
+    do_hilo2:
+        tcg_out_opc_reg(s, i1, 0, a2, args[3]);
+        tcg_out_opc_reg(s, OPC_MFLO, a0, 0, 0);
+        tcg_out_opc_reg(s, OPC_MFHI, a1, 0, 0);
+        break;
+
+    case INDEX_op_not_i32:
+        i1 = OPC_NOR;
+        goto do_unary;
+    case INDEX_op_bswap16_i32:
+        i1 = OPC_WSBH;
+        goto do_unary;
+    case INDEX_op_ext8s_i32:
+        i1 = OPC_SEB;
+        goto do_unary;
+    case INDEX_op_ext16s_i32:
+        i1 = OPC_SEH;
+    do_unary:
+        tcg_out_opc_reg(s, i1, a0, TCG_REG_ZERO, a1);
         break;
 
     case INDEX_op_sar_i32:
-        if (const_args[2]) {
-            tcg_out_opc_sa(s, OPC_SRA, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_SRAV, args[0], args[2], args[1]);
-        }
-        break;
+        i1 = OPC_SRAV, i2 = OPC_SRA;
+        goto do_shift;
     case INDEX_op_shl_i32:
-        if (const_args[2]) {
-            tcg_out_opc_sa(s, OPC_SLL, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_SLLV, args[0], args[2], args[1]);
-        }
-        break;
+        i1 = OPC_SLLV, i2 = OPC_SLL;
+        goto do_shift;
     case INDEX_op_shr_i32:
-        if (const_args[2]) {
-            tcg_out_opc_sa(s, OPC_SRL, args[0], args[1], args[2]);
+        i1 = OPC_SRLV, i2 = OPC_SRL;
+        goto do_shift;
+    case INDEX_op_rotr_i32:
+        i1 = OPC_ROTRV, i2 = OPC_ROTR;
+    do_shift:
+        if (c2) {
+            tcg_out_opc_sa(s, i2, a0, a1, a2);
         } else {
-            tcg_out_opc_reg(s, OPC_SRLV, args[0], args[2], args[1]);
+            tcg_out_opc_reg(s, i1, a0, a2, a1);
         }
         break;
     case INDEX_op_rotl_i32:
-        if (const_args[2]) {
-            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], 0x20 - args[2]);
+        if (c2) {
+            tcg_out_opc_sa(s, OPC_ROTR, a0, a1, 32 - a2);
         } else {
-            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_AT, 32);
-            tcg_out_opc_reg(s, OPC_SUBU, TCG_REG_AT, TCG_REG_AT, args[2]);
-            tcg_out_opc_reg(s, OPC_ROTRV, args[0], TCG_REG_AT, args[1]);
-        }
-        break;
-    case INDEX_op_rotr_i32:
-        if (const_args[2]) {
-            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], args[2]);
-        } else {
-            tcg_out_opc_reg(s, OPC_ROTRV, args[0], args[2], args[1]);
+            tcg_out_opc_reg(s, OPC_SUBU, TCG_TMP0, TCG_REG_ZERO, a2);
+            tcg_out_opc_reg(s, OPC_ROTRV, a0, TCG_TMP0, a1);
         }
         break;
 
-    case INDEX_op_bswap16_i32:
-        tcg_out_opc_reg(s, OPC_WSBH, args[0], 0, args[1]);
-        break;
     case INDEX_op_bswap32_i32:
-        tcg_out_opc_reg(s, OPC_WSBH, args[0], 0, args[1]);
-        tcg_out_opc_sa(s, OPC_ROTR, args[0], args[0], 16);
-        break;
-
-    case INDEX_op_ext8s_i32:
-        tcg_out_opc_reg(s, OPC_SEB, args[0], 0, args[1]);
-        break;
-    case INDEX_op_ext16s_i32:
-        tcg_out_opc_reg(s, OPC_SEH, args[0], 0, args[1]);
+        tcg_out_opc_reg(s, OPC_WSBH, a0, 0, a1);
+        tcg_out_opc_sa(s, OPC_ROTR, a0, a0, 16);
         break;
 
     case INDEX_op_deposit_i32:
-        tcg_out_opc_imm(s, OPC_INS, args[0], args[2],
-                        ((args[3] + args[4] - 1) << 11) | (args[3] << 6));
+        tcg_out_opc_bf(s, OPC_INS, a0, a2, args[3] + args[4] - 1, args[3]);
         break;
 
     case INDEX_op_brcond_i32:
-        tcg_out_brcond(s, args[2], args[0], args[1], args[3]);
+        tcg_out_brcond(s, a2, a0, a1, args[3]);
         break;
     case INDEX_op_brcond2_i32:
-        tcg_out_brcond2(s, args[4], args[0], args[1], args[2], args[3], args[5]);
+        tcg_out_brcond2(s, args[4], a0, a1, a2, args[3], args[5]);
         break;
 
     case INDEX_op_movcond_i32:
-        tcg_out_movcond(s, args[5], args[0], args[1], args[2], args[3]);
+        tcg_out_movcond(s, args[5], a0, a1, a2, args[3]);
         break;
 
     case INDEX_op_setcond_i32:
-        tcg_out_setcond(s, args[3], args[0], args[1], args[2]);
+        tcg_out_setcond(s, args[3], a0, a1, a2);
         break;
     case INDEX_op_setcond2_i32:
-        tcg_out_setcond2(s, args[5], args[0], args[1], args[2], args[3], args[4]);
+        tcg_out_setcond2(s, args[5], a0, a1, a2, args[3], args[4]);
         break;
 
-    case INDEX_op_qemu_ld8u:
-        tcg_out_qemu_ld(s, args, 0);
+    case INDEX_op_qemu_ld_i32:
+        tcg_out_qemu_ld(s, args, false);
         break;
-    case INDEX_op_qemu_ld8s:
-        tcg_out_qemu_ld(s, args, 0 | 4);
+    case INDEX_op_qemu_ld_i64:
+        tcg_out_qemu_ld(s, args, true);
         break;
-    case INDEX_op_qemu_ld16u:
-        tcg_out_qemu_ld(s, args, 1);
+    case INDEX_op_qemu_st_i32:
+        tcg_out_qemu_st(s, args, false);
         break;
-    case INDEX_op_qemu_ld16s:
-        tcg_out_qemu_ld(s, args, 1 | 4);
+    case INDEX_op_qemu_st_i64:
+        tcg_out_qemu_st(s, args, true);
         break;
-    case INDEX_op_qemu_ld32:
-        tcg_out_qemu_ld(s, args, 2);
+
+    case INDEX_op_add2_i32:
+        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
+                        const_args[4], const_args[5], false);
         break;
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-    case INDEX_op_qemu_st8:
-        tcg_out_qemu_st(s, args, 0);
-        break;
-    case INDEX_op_qemu_st16:
-        tcg_out_qemu_st(s, args, 1);
-        break;
-    case INDEX_op_qemu_st32:
-        tcg_out_qemu_st(s, args, 2);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
+    case INDEX_op_sub2_i32:
+        tcg_out_addsub2(s, a0, a1, a2, args[3], args[4], args[5],
+                        const_args[4], const_args[5], true);
         break;
 
     case INDEX_op_mov_i32:  /* Always emitted via tcg_out_mov.  */
@@ -1561,9 +1598,9 @@
     { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_remu_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rN" } },
 
-    { INDEX_op_and_i32, { "r", "rZ", "rI" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rIK" } },
     { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_not_i32, { "r", "rZ" } },
     { INDEX_op_or_i32, { "r", "rZ", "rIZ" } },
@@ -1588,34 +1625,20 @@
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rN", "rN" } },
     { INDEX_op_brcond2_i32, { "rZ", "rZ", "rZ", "rZ" } },
 
 #if TARGET_LONG_BITS == 32
-    { INDEX_op_qemu_ld8u, { "L", "lZ" } },
-    { INDEX_op_qemu_ld8s, { "L", "lZ" } },
-    { INDEX_op_qemu_ld16u, { "L", "lZ" } },
-    { INDEX_op_qemu_ld16s, { "L", "lZ" } },
-    { INDEX_op_qemu_ld32, { "L", "lZ" } },
-    { INDEX_op_qemu_ld64, { "L", "L", "lZ" } },
-
-    { INDEX_op_qemu_st8, { "SZ", "SZ" } },
-    { INDEX_op_qemu_st16, { "SZ", "SZ" } },
-    { INDEX_op_qemu_st32, { "SZ", "SZ" } },
-    { INDEX_op_qemu_st64, { "SZ", "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i32, { "L", "lZ" } },
+    { INDEX_op_qemu_st_i32, { "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ" } },
 #else
-    { INDEX_op_qemu_ld8u, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_ld8s, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_ld16u, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_ld16s, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_ld32, { "L", "lZ", "lZ" } },
-    { INDEX_op_qemu_ld64, { "L", "L", "lZ", "lZ" } },
-
-    { INDEX_op_qemu_st8, { "SZ", "SZ", "SZ" } },
-    { INDEX_op_qemu_st16, { "SZ", "SZ", "SZ" } },
-    { INDEX_op_qemu_st32, { "SZ", "SZ", "SZ" } },
-    { INDEX_op_qemu_st64, { "SZ", "SZ", "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i32, { "L", "lZ", "lZ" } },
+    { INDEX_op_qemu_st_i32, { "SZ", "SZ", "SZ" } },
+    { INDEX_op_qemu_ld_i64, { "L", "L", "lZ", "lZ" } },
+    { INDEX_op_qemu_st_i64, { "SZ", "SZ", "SZ", "SZ" } },
 #endif
     { -1 },
 };
@@ -1629,7 +1652,7 @@
     TCG_REG_S5,
     TCG_REG_S6,
     TCG_REG_S7,
-    TCG_REG_FP,
+    TCG_REG_S8,
     TCG_REG_RA,       /* should be last for ABI compliance */
 };
 
@@ -1761,6 +1784,7 @@
                    (1 << TCG_REG_A1) |
                    (1 << TCG_REG_A2) |
                    (1 << TCG_REG_A3) |
+                   (1 << TCG_REG_T0) |
                    (1 << TCG_REG_T1) |
                    (1 << TCG_REG_T2) |
                    (1 << TCG_REG_T3) |
@@ -1775,11 +1799,18 @@
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_ZERO); /* zero register */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_K0);   /* kernel use only */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_K1);   /* kernel use only */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_AT);   /* internal use */
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T0);   /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP0);     /* internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_TMP1);     /* internal use */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);   /* return address */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);   /* stack pointer */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
 
     tcg_add_target_add_op_defs(mips_op_defs);
 }
+
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
+{
+    uint32_t *ptr = (uint32_t *)jmp_addr;
+    *ptr = deposit32(*ptr, 0, 26, addr >> 2);
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index c6d2267..c88a1c9 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -60,16 +60,14 @@
     TCG_REG_K1,
     TCG_REG_GP,
     TCG_REG_SP,
-    TCG_REG_FP,
+    TCG_REG_S8,
     TCG_REG_RA,
+
+    TCG_REG_CALL_STACK = TCG_REG_SP,
+    TCG_AREG0 = TCG_REG_S0,
 } TCGReg;
 
-#define TCG_CT_CONST_ZERO 0x100
-#define TCG_CT_CONST_U16  0x200
-#define TCG_CT_CONST_S16  0x400
-
 /* used for function call generation */
-#define TCG_REG_CALL_STACK TCG_REG_SP
 #define TCG_TARGET_STACK_ALIGN 8
 #define TCG_TARGET_CALL_STACK_OFFSET 16
 #define TCG_TARGET_CALL_ALIGN_ARGS 1
@@ -120,15 +118,11 @@
 #define TCG_TARGET_HAS_ext16s_i32       use_mips32r2_instructions
 #define TCG_TARGET_HAS_rot_i32          use_mips32r2_instructions
 
-#define TCG_TARGET_HAS_new_ldst         0
-
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub  rd, zero, rt   */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* andi rt, rs, 0xff   */
 #define TCG_TARGET_HAS_ext16u_i32       0 /* andi rt, rs, 0xffff */
 
-#define TCG_AREG0 TCG_REG_S0
-
 #ifdef __OpenBSD__
 #include <machine/sysarch.h>
 #else
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 3a504a1..16cebbe 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -83,6 +83,20 @@
     return def->flags & TCG_OPF_64BIT ? 64 : 32;
 }
 
+static TCGOpcode op_to_mov(TCGOpcode op)
+{
+    switch (op_bits(op)) {
+    case 32:
+        return INDEX_op_mov_i32;
+    case 64:
+        return INDEX_op_mov_i64;
+    default:
+        fprintf(stderr, "op_to_mov: unexpected return value of "
+                "function op_bits.\n");
+        tcg_abort();
+    }
+}
+
 static TCGOpcode op_to_movi(TCGOpcode op)
 {
     switch (op_bits(op)) {
@@ -148,11 +162,22 @@
     return false;
 }
 
-static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args,
-                            TCGArg dst, TCGArg src)
+static void tcg_opt_gen_mov(TCGContext *s, int op_index, TCGArg *gen_args,
+                            TCGOpcode old_op, TCGArg dst, TCGArg src)
 {
+    TCGOpcode new_op = op_to_mov(old_op);
+    tcg_target_ulong mask;
+
+    s->gen_opc_buf[op_index] = new_op;
+
     reset_temp(dst);
-    temps[dst].mask = temps[src].mask;
+    mask = temps[src].mask;
+    if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) {
+        /* High bits of the destination are now garbage.  */
+        mask |= ~0xffffffffull;
+    }
+    temps[dst].mask = mask;
+
     assert(temps[src].state != TCG_TEMP_CONST);
 
     if (s->temps[src].type == s->temps[dst].type) {
@@ -172,30 +197,28 @@
     gen_args[1] = src;
 }
 
-static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val)
+static void tcg_opt_gen_movi(TCGContext *s, int op_index, TCGArg *gen_args,
+                             TCGOpcode old_op, TCGArg dst, TCGArg val)
 {
+    TCGOpcode new_op = op_to_movi(old_op);
+    tcg_target_ulong mask;
+
+    s->gen_opc_buf[op_index] = new_op;
+
     reset_temp(dst);
     temps[dst].state = TCG_TEMP_CONST;
     temps[dst].val = val;
-    temps[dst].mask = val;
+    mask = val;
+    if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) {
+        /* High bits of the destination are now garbage.  */
+        mask |= ~0xffffffffull;
+    }
+    temps[dst].mask = mask;
+
     gen_args[0] = dst;
     gen_args[1] = val;
 }
 
-static TCGOpcode op_to_mov(TCGOpcode op)
-{
-    switch (op_bits(op)) {
-    case 32:
-        return INDEX_op_mov_i32;
-    case 64:
-        return INDEX_op_mov_i64;
-    default:
-        fprintf(stderr, "op_to_mov: unexpected return value of "
-                "function op_bits.\n");
-        tcg_abort();
-    }
-}
-
 static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
 {
     uint64_t l64, h64;
@@ -530,7 +553,7 @@
     for (op_index = 0; op_index < nb_ops; op_index++) {
         TCGOpcode op = s->gen_opc_buf[op_index];
         const TCGOpDef *def = &tcg_op_defs[op];
-        tcg_target_ulong mask, affected;
+        tcg_target_ulong mask, partmask, affected;
         int nb_oargs, nb_iargs, nb_args, i;
         TCGArg tmp;
 
@@ -619,8 +642,7 @@
         CASE_OP_32_64(rotr):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[1]].val == 0) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], 0);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], 0);
                 args += 3;
                 gen_args += 2;
                 continue;
@@ -749,8 +771,7 @@
             if (temps_are_copies(args[0], args[1])) {
                 s->gen_opc_buf[op_index] = INDEX_op_nop;
             } else {
-                s->gen_opc_buf[op_index] = op_to_mov(op);
-                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                tcg_opt_gen_mov(s, op_index, gen_args, op, args[0], args[1]);
                 gen_args += 2;
             }
             args += 3;
@@ -859,6 +880,7 @@
             break;
 
         CASE_OP_32_64(setcond):
+        case INDEX_op_setcond2_i32:
             mask = 1;
             break;
 
@@ -867,17 +889,12 @@
             break;
 
         CASE_OP_32_64(ld8u):
-        case INDEX_op_qemu_ld8u:
             mask = 0xff;
             break;
         CASE_OP_32_64(ld16u):
-        case INDEX_op_qemu_ld16u:
             mask = 0xffff;
             break;
         case INDEX_op_ld32u_i64:
-#if TCG_TARGET_REG_BITS == 64
-        case INDEX_op_qemu_ld32u:
-#endif
             mask = 0xffffffffu;
             break;
 
@@ -894,16 +911,20 @@
             break;
         }
 
-        /* 32-bit ops (non 64-bit ops and non load/store ops) generate 32-bit
-           results */
+        /* 32-bit ops (non 64-bit ops and non load/store ops) generate
+           32-bit results.  For the result is zero test below, we can
+           ignore high bits, but for further optimizations we need to
+           record that the high bits contain garbage.  */
+        partmask = mask;
         if (!(def->flags & (TCG_OPF_CALL_CLOBBER | TCG_OPF_64BIT))) {
-            mask &= 0xffffffffu;
+            mask |= ~(tcg_target_ulong)0xffffffffu;
+            partmask &= 0xffffffffu;
+            affected &= 0xffffffffu;
         }
 
-        if (mask == 0) {
+        if (partmask == 0) {
             assert(nb_oargs == 1);
-            s->gen_opc_buf[op_index] = op_to_movi(op);
-            tcg_opt_gen_movi(gen_args, args[0], 0);
+            tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], 0);
             args += nb_args;
             gen_args += 2;
             continue;
@@ -913,12 +934,11 @@
             if (temps_are_copies(args[0], args[1])) {
                 s->gen_opc_buf[op_index] = INDEX_op_nop;
             } else if (temps[args[1]].state != TCG_TEMP_CONST) {
-                s->gen_opc_buf[op_index] = op_to_mov(op);
-                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                tcg_opt_gen_mov(s, op_index, gen_args, op, args[0], args[1]);
                 gen_args += 2;
             } else {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], temps[args[1]].val);
+                tcg_opt_gen_movi(s, op_index, gen_args, op,
+                                 args[0], temps[args[1]].val);
                 gen_args += 2;
             }
             args += nb_args;
@@ -933,8 +953,7 @@
         CASE_OP_32_64(mulsh):
             if ((temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0)) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], 0);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], 0);
                 args += 3;
                 gen_args += 2;
                 continue;
@@ -952,8 +971,8 @@
                 if (temps_are_copies(args[0], args[1])) {
                     s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else {
-                    s->gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                    tcg_opt_gen_mov(s, op_index, gen_args, op,
+                                    args[0], args[1]);
                     gen_args += 2;
                 }
                 args += 3;
@@ -970,8 +989,7 @@
         CASE_OP_32_64(sub):
         CASE_OP_32_64(xor):
             if (temps_are_copies(args[1], args[2])) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], 0);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], 0);
                 gen_args += 2;
                 args += 3;
                 continue;
@@ -992,19 +1010,17 @@
                 break;
             }
             if (temps[args[1]].state != TCG_TEMP_CONST) {
-                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
+                tcg_opt_gen_mov(s, op_index, gen_args, op, args[0], args[1]);
                 gen_args += 2;
                 args += 2;
                 break;
             }
             /* Source argument is constant.  Rewrite the operation and
                let movi case handle it. */
-            op = op_to_movi(op);
-            s->gen_opc_buf[op_index] = op;
             args[1] = temps[args[1]].val;
             /* fallthrough */
         CASE_OP_32_64(movi):
-            tcg_opt_gen_movi(gen_args, args[0], args[1]);
+            tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], args[1]);
             gen_args += 2;
             args += 2;
             break;
@@ -1018,9 +1034,8 @@
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
             if (temps[args[1]].state == TCG_TEMP_CONST) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val, 0);
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
                 args += 2;
                 break;
@@ -1029,9 +1044,8 @@
 
         case INDEX_op_trunc_shr_i32:
             if (temps[args[1]].state == TCG_TEMP_CONST) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val, args[2]);
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
                 args += 3;
                 break;
@@ -1062,10 +1076,9 @@
         CASE_OP_32_64(remu):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[2]].state == TCG_TEMP_CONST) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val,
                                           temps[args[2]].val);
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
                 args += 3;
                 break;
@@ -1075,10 +1088,9 @@
         CASE_OP_32_64(deposit):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[2]].state == TCG_TEMP_CONST) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = deposit64(temps[args[1]].val, args[3], args[4],
                                 temps[args[2]].val);
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
                 args += 5;
                 break;
@@ -1088,8 +1100,7 @@
         CASE_OP_32_64(setcond):
             tmp = do_constant_folding_cond(op, args[1], args[2], args[3]);
             if (tmp != 2) {
-                s->gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
                 args += 4;
                 break;
@@ -1118,12 +1129,12 @@
                 if (temps_are_copies(args[0], args[4-tmp])) {
                     s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else if (temps[args[4-tmp]].state == TCG_TEMP_CONST) {
-                    s->gen_opc_buf[op_index] = op_to_movi(op);
-                    tcg_opt_gen_movi(gen_args, args[0], temps[args[4-tmp]].val);
+                    tcg_opt_gen_movi(s, op_index, gen_args, op,
+                                     args[0], temps[args[4-tmp]].val);
                     gen_args += 2;
                 } else {
-                    s->gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[4-tmp]);
+                    tcg_opt_gen_mov(s, op_index, gen_args, op,
+                                    args[0], args[4-tmp]);
                     gen_args += 2;
                 }
                 args += 6;
@@ -1156,10 +1167,10 @@
 
                 rl = args[0];
                 rh = args[1];
-                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
-                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
-                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)a);
-                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(a >> 32));
+                tcg_opt_gen_movi(s, op_index, &gen_args[0],
+                                 op, rl, (uint32_t)a);
+                tcg_opt_gen_movi(s, ++op_index, &gen_args[2],
+                                 op, rh, (uint32_t)(a >> 32));
                 gen_args += 4;
                 args += 6;
                 break;
@@ -1179,10 +1190,10 @@
 
                 rl = args[0];
                 rh = args[1];
-                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
-                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
-                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)r);
-                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(r >> 32));
+                tcg_opt_gen_movi(s, op_index, &gen_args[0],
+                                 op, rl, (uint32_t)r);
+                tcg_opt_gen_movi(s, ++op_index, &gen_args[2],
+                                 op, rh, (uint32_t)(r >> 32));
                 gen_args += 4;
                 args += 4;
                 break;
@@ -1193,11 +1204,13 @@
             tmp = do_constant_folding_cond2(&args[0], &args[2], args[4]);
             if (tmp != 2) {
                 if (tmp) {
+            do_brcond_true:
                     reset_all_temps(nb_temps);
                     s->gen_opc_buf[op_index] = INDEX_op_br;
                     gen_args[0] = args[5];
                     gen_args += 1;
                 } else {
+            do_brcond_false:
                     s->gen_opc_buf[op_index] = INDEX_op_nop;
                 }
             } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
@@ -1207,6 +1220,7 @@
                        && temps[args[3]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
+            do_brcond_high:
                 reset_all_temps(nb_temps);
                 s->gen_opc_buf[op_index] = INDEX_op_brcond_i32;
                 gen_args[0] = args[1];
@@ -1214,6 +1228,49 @@
                 gen_args[2] = args[4];
                 gen_args[3] = args[5];
                 gen_args += 4;
+            } else if (args[4] == TCG_COND_EQ) {
+                /* Simplify EQ comparisons where one of the pairs
+                   can be simplified.  */
+                tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
+                                               args[0], args[2], TCG_COND_EQ);
+                if (tmp == 0) {
+                    goto do_brcond_false;
+                } else if (tmp == 1) {
+                    goto do_brcond_high;
+                }
+                tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
+                                               args[1], args[3], TCG_COND_EQ);
+                if (tmp == 0) {
+                    goto do_brcond_false;
+                } else if (tmp != 1) {
+                    goto do_default;
+                }
+            do_brcond_low:
+                reset_all_temps(nb_temps);
+                s->gen_opc_buf[op_index] = INDEX_op_brcond_i32;
+                gen_args[0] = args[0];
+                gen_args[1] = args[2];
+                gen_args[2] = args[4];
+                gen_args[3] = args[5];
+                gen_args += 4;
+            } else if (args[4] == TCG_COND_NE) {
+                /* Simplify NE comparisons where one of the pairs
+                   can be simplified.  */
+                tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
+                                               args[0], args[2], TCG_COND_NE);
+                if (tmp == 0) {
+                    goto do_brcond_high;
+                } else if (tmp == 1) {
+                    goto do_brcond_true;
+                }
+                tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
+                                               args[1], args[3], TCG_COND_NE);
+                if (tmp == 0) {
+                    goto do_brcond_low;
+                } else if (tmp == 1) {
+                    goto do_brcond_true;
+                }
+                goto do_default;
             } else {
                 goto do_default;
             }
@@ -1223,8 +1280,8 @@
         case INDEX_op_setcond2_i32:
             tmp = do_constant_folding_cond2(&args[1], &args[3], args[5]);
             if (tmp != 2) {
-                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
-                tcg_opt_gen_movi(gen_args, args[0], tmp);
+            do_setcond_const:
+                tcg_opt_gen_movi(s, op_index, gen_args, op, args[0], tmp);
                 gen_args += 2;
             } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
                        && temps[args[3]].state == TCG_TEMP_CONST
@@ -1233,13 +1290,59 @@
                        && temps[args[4]].val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
+            do_setcond_high:
                 s->gen_opc_buf[op_index] = INDEX_op_setcond_i32;
                 reset_temp(args[0]);
+                temps[args[0]].mask = 1;
                 gen_args[0] = args[0];
                 gen_args[1] = args[2];
                 gen_args[2] = args[4];
                 gen_args[3] = args[5];
                 gen_args += 4;
+            } else if (args[5] == TCG_COND_EQ) {
+                /* Simplify EQ comparisons where one of the pairs
+                   can be simplified.  */
+                tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
+                                               args[1], args[3], TCG_COND_EQ);
+                if (tmp == 0) {
+                    goto do_setcond_const;
+                } else if (tmp == 1) {
+                    goto do_setcond_high;
+                }
+                tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
+                                               args[2], args[4], TCG_COND_EQ);
+                if (tmp == 0) {
+                    goto do_setcond_high;
+                } else if (tmp != 1) {
+                    goto do_default;
+                }
+            do_setcond_low:
+                reset_temp(args[0]);
+                temps[args[0]].mask = 1;
+                s->gen_opc_buf[op_index] = INDEX_op_setcond_i32;
+                gen_args[0] = args[0];
+                gen_args[1] = args[1];
+                gen_args[2] = args[3];
+                gen_args[3] = args[5];
+                gen_args += 4;
+            } else if (args[5] == TCG_COND_NE) {
+                /* Simplify NE comparisons where one of the pairs
+                   can be simplified.  */
+                tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
+                                               args[1], args[3], TCG_COND_NE);
+                if (tmp == 0) {
+                    goto do_setcond_high;
+                } else if (tmp == 1) {
+                    goto do_setcond_const;
+                }
+                tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
+                                               args[2], args[4], TCG_COND_NE);
+                if (tmp == 0) {
+                    goto do_setcond_low;
+                } else if (tmp == 1) {
+                    goto do_setcond_const;
+                }
+                goto do_default;
             } else {
                 goto do_default;
             }
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index dd7e557..05069ae 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -100,8 +100,6 @@
 #define TCG_TARGET_HAS_muluh_i32        0
 #define TCG_TARGET_HAS_mulsh_i32        0
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 #define TCG_AREG0 TCG_REG_R27
 
 #define tcg_qemu_tb_exec(env, tb_ptr) \
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index 29f479a..f2360c8 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -124,8 +124,6 @@
 #define TCG_TARGET_HAS_muluh_i64        1
 #define TCG_TARGET_HAS_mulsh_i64        1
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 #define TCG_AREG0 TCG_REG_R27
 
 #define TCG_TARGET_EXTEND_ARGS 1
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index 07164e5..63e9c82 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -2344,8 +2344,7 @@
 }
 
 typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
+    DebugFrameHeader h;
     uint8_t fde_def_cfa[4];
     uint8_t fde_reg_ofs[18];
 } DebugFrame;
@@ -2355,16 +2354,16 @@
 
 #define ELF_HOST_MACHINE  EM_S390
 
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = 8,                /* sleb128 8 */
-    .cie.return_column = TCG_REG_R14,
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = 8,                /* sleb128 8 */
+    .h.cie.return_column = TCG_REG_R14,
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
         12, TCG_REG_CALL_STACK,         /* DW_CFA_def_cfa %r15, ... */
@@ -2386,8 +2385,5 @@
 
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (uintptr_t)buf;
-    debug_frame.fde.func_len = buf_size;
-
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index ad2c6dd..5acc28c 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -100,8 +100,6 @@
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i64        0
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 extern bool tcg_target_deposit_valid(int ofs, int len);
 #define TCG_TARGET_deposit_i32_valid  tcg_target_deposit_valid
 #define TCG_TARGET_deposit_i64_valid  tcg_target_deposit_valid
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 17ff577..40f2ec1 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -1499,23 +1499,22 @@
 #endif
 
 typedef struct {
-    DebugFrameCIE cie;
-    DebugFrameFDEHeader fde;
+    DebugFrameHeader h;
     uint8_t fde_def_cfa[SPARC64 ? 4 : 2];
     uint8_t fde_win_save;
     uint8_t fde_ret_save[3];
 } DebugFrame;
 
-static DebugFrame debug_frame = {
-    .cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
-    .cie.id = -1,
-    .cie.version = 1,
-    .cie.code_align = 1,
-    .cie.data_align = -sizeof(void *) & 0x7f,
-    .cie.return_column = 15,            /* o7 */
+static const DebugFrame debug_frame = {
+    .h.cie.len = sizeof(DebugFrameCIE)-4, /* length after .len member */
+    .h.cie.id = -1,
+    .h.cie.version = 1,
+    .h.cie.code_align = 1,
+    .h.cie.data_align = -sizeof(void *) & 0x7f,
+    .h.cie.return_column = 15,            /* o7 */
 
     /* Total FDE size does not include the "len" member.  */
-    .fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, fde.cie_offset),
+    .h.fde.len = sizeof(DebugFrame) - offsetof(DebugFrame, h.fde.cie_offset),
 
     .fde_def_cfa = {
 #if SPARC64
@@ -1531,9 +1530,6 @@
 
 void tcg_register_jit(void *buf, size_t buf_size)
 {
-    debug_frame.fde.func_start = (uintptr_t)buf;
-    debug_frame.fde.func_len = buf_size;
-
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
 
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index 473bfc7..089f976 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -140,8 +140,6 @@
 #define TCG_TARGET_HAS_muluh_i64        0
 #define TCG_TARGET_HAS_mulsh_i64        0
 
-#define TCG_TARGET_HAS_new_ldst         1
-
 #define TCG_AREG0 TCG_REG_I0
 
 static inline void flush_icache_range(uintptr_t start, uintptr_t stop)
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index bdd0139..019dd9b 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 #include "tcg.h"
+#include "exec/helper-proto.h"
+#include "exec/helper-gen.h"
 
 int gen_new_label(void);
 
@@ -379,47 +381,6 @@
     tcg_gen_op2i_i32(INDEX_op_movi_i32, ret, arg);
 }
 
-/* A version of dh_sizemask from def-helper.h that doesn't rely on
-   preprocessor magic.  */
-static inline int tcg_gen_sizemask(int n, int is_64bit, int is_signed)
-{
-    return (is_64bit << n*2) | (is_signed << (n*2 + 1));
-}
-
-/* helper calls */
-static inline void tcg_gen_helperN(void *func, int flags, int sizemask,
-                                   TCGArg ret, int nargs, TCGArg *args)
-{
-    tcg_gen_callN(&tcg_ctx, func, flags, sizemask, ret, nargs, args);
-}
-
-/* Note: Both tcg_gen_helper32() and tcg_gen_helper64() are currently
-   reserved for helpers in tcg-runtime.c. These helpers all do not read
-   globals and do not have side effects, hence the call to tcg_gen_callN()
-   with TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS. This may need
-   to be adjusted if these functions start to be used with other helpers. */
-static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
-                                    TCGv_i32 a, TCGv_i32 b)
-{
-    TCGArg args[2];
-    args[0] = GET_TCGV_I32(a);
-    args[1] = GET_TCGV_I32(b);
-    tcg_gen_callN(&tcg_ctx, func,
-                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
-                  sizemask, GET_TCGV_I32(ret), 2, args);
-}
-
-static inline void tcg_gen_helper64(void *func, int sizemask, TCGv_i64 ret,
-                                    TCGv_i64 a, TCGv_i64 b)
-{
-    TCGArg args[2];
-    args[0] = GET_TCGV_I64(a);
-    args[1] = GET_TCGV_I64(b);
-    tcg_gen_callN(&tcg_ctx, func,
-                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
-                  sizemask, GET_TCGV_I64(ret), 2, args);
-}
-
 /* 32 bit ops */
 
 static inline void tcg_gen_ld8u_i32(TCGv_i32 ret, TCGv_ptr arg2, tcg_target_long offset)
@@ -707,12 +668,7 @@
         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
         tcg_temp_free_i32(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 32-bit and signed.  */
-        sizemask |= tcg_gen_sizemask(0, 0, 1);
-        sizemask |= tcg_gen_sizemask(1, 0, 1);
-        sizemask |= tcg_gen_sizemask(2, 0, 1);
-        tcg_gen_helper32(tcg_helper_div_i32, sizemask, ret, arg1, arg2);
+        gen_helper_div_i32(ret, arg1, arg2);
     }
 }
 
@@ -732,12 +688,7 @@
         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
         tcg_temp_free_i32(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 32-bit and signed.  */
-        sizemask |= tcg_gen_sizemask(0, 0, 1);
-        sizemask |= tcg_gen_sizemask(1, 0, 1);
-        sizemask |= tcg_gen_sizemask(2, 0, 1);
-        tcg_gen_helper32(tcg_helper_rem_i32, sizemask, ret, arg1, arg2);
+        gen_helper_rem_i32(ret, arg1, arg2);
     }
 }
 
@@ -751,12 +702,7 @@
         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
         tcg_temp_free_i32(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 32-bit and unsigned.  */
-        sizemask |= tcg_gen_sizemask(0, 0, 0);
-        sizemask |= tcg_gen_sizemask(1, 0, 0);
-        sizemask |= tcg_gen_sizemask(2, 0, 0);
-        tcg_gen_helper32(tcg_helper_divu_i32, sizemask, ret, arg1, arg2);
+        gen_helper_divu_i32(ret, arg1, arg2);
     }
 }
 
@@ -776,12 +722,7 @@
         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
         tcg_temp_free_i32(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 32-bit and unsigned.  */
-        sizemask |= tcg_gen_sizemask(0, 0, 0);
-        sizemask |= tcg_gen_sizemask(1, 0, 0);
-        sizemask |= tcg_gen_sizemask(2, 0, 0);
-        tcg_gen_helper32(tcg_helper_remu_i32, sizemask, ret, arg1, arg2);
+        gen_helper_remu_i32(ret, arg1, arg2);
     }
 }
 
@@ -945,13 +886,7 @@
    specific code (x86) */
 static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and signed.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 1);
-    sizemask |= tcg_gen_sizemask(1, 1, 1);
-    sizemask |= tcg_gen_sizemask(2, 1, 1);
-
-    tcg_gen_helper64(tcg_helper_shl_i64, sizemask, ret, arg1, arg2);
+    gen_helper_shl_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
@@ -961,13 +896,7 @@
 
 static inline void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and signed.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 1);
-    sizemask |= tcg_gen_sizemask(1, 1, 1);
-    sizemask |= tcg_gen_sizemask(2, 1, 1);
-
-    tcg_gen_helper64(tcg_helper_shr_i64, sizemask, ret, arg1, arg2);
+    gen_helper_shr_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
@@ -977,13 +906,7 @@
 
 static inline void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and signed.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 1);
-    sizemask |= tcg_gen_sizemask(1, 1, 1);
-    sizemask |= tcg_gen_sizemask(2, 1, 1);
-
-    tcg_gen_helper64(tcg_helper_sar_i64, sizemask, ret, arg1, arg2);
+    gen_helper_sar_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
@@ -1051,46 +974,22 @@
 
 static inline void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and signed.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 1);
-    sizemask |= tcg_gen_sizemask(1, 1, 1);
-    sizemask |= tcg_gen_sizemask(2, 1, 1);
-
-    tcg_gen_helper64(tcg_helper_div_i64, sizemask, ret, arg1, arg2);
+    gen_helper_div_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and signed.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 1);
-    sizemask |= tcg_gen_sizemask(1, 1, 1);
-    sizemask |= tcg_gen_sizemask(2, 1, 1);
-
-    tcg_gen_helper64(tcg_helper_rem_i64, sizemask, ret, arg1, arg2);
+    gen_helper_rem_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and unsigned.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 0);
-    sizemask |= tcg_gen_sizemask(1, 1, 0);
-    sizemask |= tcg_gen_sizemask(2, 1, 0);
-
-    tcg_gen_helper64(tcg_helper_divu_i64, sizemask, ret, arg1, arg2);
+    gen_helper_divu_i64(ret, arg1, arg2);
 }
 
 static inline void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    int sizemask = 0;
-    /* Return value and both arguments are 64-bit and unsigned.  */
-    sizemask |= tcg_gen_sizemask(0, 1, 0);
-    sizemask |= tcg_gen_sizemask(1, 1, 0);
-    sizemask |= tcg_gen_sizemask(2, 1, 0);
-
-    tcg_gen_helper64(tcg_helper_remu_i64, sizemask, ret, arg1, arg2);
+    gen_helper_remu_i64(ret, arg1, arg2);
 }
 
 #else
@@ -1357,12 +1256,7 @@
         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
         tcg_temp_free_i64(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and signed.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 1);
-        sizemask |= tcg_gen_sizemask(1, 1, 1);
-        sizemask |= tcg_gen_sizemask(2, 1, 1);
-        tcg_gen_helper64(tcg_helper_div_i64, sizemask, ret, arg1, arg2);
+        gen_helper_div_i64(ret, arg1, arg2);
     }
 }
 
@@ -1382,12 +1276,7 @@
         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
         tcg_temp_free_i64(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and signed.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 1);
-        sizemask |= tcg_gen_sizemask(1, 1, 1);
-        sizemask |= tcg_gen_sizemask(2, 1, 1);
-        tcg_gen_helper64(tcg_helper_rem_i64, sizemask, ret, arg1, arg2);
+        gen_helper_rem_i64(ret, arg1, arg2);
     }
 }
 
@@ -1401,12 +1290,7 @@
         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
         tcg_temp_free_i64(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and unsigned.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 0);
-        sizemask |= tcg_gen_sizemask(1, 1, 0);
-        sizemask |= tcg_gen_sizemask(2, 1, 0);
-        tcg_gen_helper64(tcg_helper_divu_i64, sizemask, ret, arg1, arg2);
+        gen_helper_divu_i64(ret, arg1, arg2);
     }
 }
 
@@ -1426,12 +1310,7 @@
         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
         tcg_temp_free_i64(t0);
     } else {
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and unsigned.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 0);
-        sizemask |= tcg_gen_sizemask(1, 1, 0);
-        sizemask |= tcg_gen_sizemask(2, 1, 0);
-        tcg_gen_helper64(tcg_helper_remu_i64, sizemask, ret, arg1, arg2);
+        gen_helper_remu_i64(ret, arg1, arg2);
     }
 }
 #endif /* TCG_TARGET_REG_BITS == 32 */
@@ -2530,13 +2409,8 @@
         tcg_temp_free_i64(t);
     } else {
         TCGv_i64 t0 = tcg_temp_new_i64();
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and unsigned.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 0);
-        sizemask |= tcg_gen_sizemask(1, 1, 0);
-        sizemask |= tcg_gen_sizemask(2, 1, 0);
         tcg_gen_mul_i64(t0, arg1, arg2);
-        tcg_gen_helper64(tcg_helper_muluh_i64, sizemask, rh, arg1, arg2);
+        gen_helper_muluh_i64(rh, arg1, arg2);
         tcg_gen_mov_i64(rl, t0);
         tcg_temp_free_i64(t0);
     }
@@ -2575,13 +2449,8 @@
         tcg_temp_free_i64(t3);
     } else {
         TCGv_i64 t0 = tcg_temp_new_i64();
-        int sizemask = 0;
-        /* Return value and both arguments are 64-bit and signed.  */
-        sizemask |= tcg_gen_sizemask(0, 1, 1);
-        sizemask |= tcg_gen_sizemask(1, 1, 1);
-        sizemask |= tcg_gen_sizemask(2, 1, 1);
         tcg_gen_mul_i64(t0, arg1, arg2);
-        tcg_gen_helper64(tcg_helper_mulsh_i64, sizemask, rh, arg1, arg2);
+        gen_helper_mulsh_i64(rh, arg1, arg2);
         tcg_gen_mov_i64(rl, t0);
         tcg_temp_free_i64(t0);
     }
@@ -2849,7 +2718,7 @@
 #define tcg_gen_bswap16_tl tcg_gen_bswap16_i32
 #define tcg_gen_bswap32_tl tcg_gen_bswap32_i32
 #define tcg_gen_concat_tl_i64 tcg_gen_concat_i32_i64
-#define tcg_gen_extr_tl_i64 tcg_gen_extr_i32_i64
+#define tcg_gen_extr_i64_tl tcg_gen_extr_i64_i32
 #define tcg_gen_andc_tl tcg_gen_andc_i32
 #define tcg_gen_eqv_tl tcg_gen_eqv_i32
 #define tcg_gen_nand_tl tcg_gen_nand_i32
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index 71ba64a..042d442 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -185,106 +185,20 @@
 DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
 DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 
-#define IMPL_NEW_LDST \
-    (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS \
-     | IMPL(TCG_TARGET_HAS_new_ldst))
+#define TLADDR_ARGS    (TARGET_LONG_BITS <= TCG_TARGET_REG_BITS ? 1 : 2)
+#define DATA64_ARGS  (TCG_TARGET_REG_BITS == 64 ? 1 : 2)
 
-#if TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
-DEF(qemu_ld_i32, 1, 1, 2, IMPL_NEW_LDST)
-DEF(qemu_st_i32, 0, 2, 2, IMPL_NEW_LDST)
-# if TCG_TARGET_REG_BITS == 64
-DEF(qemu_ld_i64, 1, 1, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-DEF(qemu_st_i64, 0, 2, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-# else
-DEF(qemu_ld_i64, 2, 1, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-DEF(qemu_st_i64, 0, 3, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-# endif
-#else
-DEF(qemu_ld_i32, 1, 2, 2, IMPL_NEW_LDST)
-DEF(qemu_st_i32, 0, 3, 2, IMPL_NEW_LDST)
-DEF(qemu_ld_i64, 2, 2, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-DEF(qemu_st_i64, 0, 4, 2, IMPL_NEW_LDST | TCG_OPF_64BIT)
-#endif
+DEF(qemu_ld_i32, 1, TLADDR_ARGS, 2,
+    TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_st_i32, 0, TLADDR_ARGS + 1, 2,
+    TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS)
+DEF(qemu_ld_i64, DATA64_ARGS, TLADDR_ARGS, 2,
+    TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT)
+DEF(qemu_st_i64, 0, TLADDR_ARGS + DATA64_ARGS, 2,
+    TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS | TCG_OPF_64BIT)
 
-#undef IMPL_NEW_LDST
-
-#define IMPL_OLD_LDST \
-    (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS \
-     | IMPL(!TCG_TARGET_HAS_new_ldst))
-
-#if TCG_TARGET_REG_BITS == 32
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld8u, 1, 1, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_ld8u, 1, 2, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld8s, 1, 1, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_ld8s, 1, 2, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld16u, 1, 1, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_ld16u, 1, 2, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld16s, 1, 1, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_ld16s, 1, 2, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld32, 1, 1, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_ld32, 1, 2, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_ld64, 2, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-#else
-DEF(qemu_ld64, 2, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-#endif
-
-#if TARGET_LONG_BITS == 32
-DEF(qemu_st8, 0, 2, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_st8, 0, 3, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_st16, 0, 2, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_st16, 0, 3, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_st32, 0, 2, 1, IMPL_OLD_LDST)
-#else
-DEF(qemu_st32, 0, 3, 1, IMPL_OLD_LDST)
-#endif
-#if TARGET_LONG_BITS == 32
-DEF(qemu_st64, 0, 3, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-#else
-DEF(qemu_st64, 0, 4, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-#endif
-
-#else /* TCG_TARGET_REG_BITS == 32 */
-
-DEF(qemu_ld8u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld8s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld16u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld16s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld32, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld32u, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld32s, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_ld64, 1, 1, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-
-DEF(qemu_st8, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_st16, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_st32, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-DEF(qemu_st64, 0, 2, 1, IMPL_OLD_LDST | TCG_OPF_64BIT)
-
-#endif /* TCG_TARGET_REG_BITS != 32 */
-
-#undef IMPL_OLD_LDST
-
+#undef TLADDR_ARGS
+#undef DATA64_ARGS
 #undef IMPL
 #undef IMPL64
 #undef DEF
diff --git a/tcg/tcg-runtime.h b/tcg/tcg-runtime.h
index a1ebef9..23a0c37 100644
--- a/tcg/tcg-runtime.h
+++ b/tcg/tcg-runtime.h
@@ -1,20 +1,16 @@
-#ifndef TCG_RUNTIME_H
-#define TCG_RUNTIME_H
+DEF_HELPER_FLAGS_2(div_i32, TCG_CALL_NO_RWG_SE, s32, s32, s32)
+DEF_HELPER_FLAGS_2(rem_i32, TCG_CALL_NO_RWG_SE, s32, s32, s32)
+DEF_HELPER_FLAGS_2(divu_i32, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_2(remu_i32, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 
-/* tcg-runtime.c */
-int32_t tcg_helper_div_i32(int32_t arg1, int32_t arg2);
-int32_t tcg_helper_rem_i32(int32_t arg1, int32_t arg2);
-uint32_t tcg_helper_divu_i32(uint32_t arg1, uint32_t arg2);
-uint32_t tcg_helper_remu_i32(uint32_t arg1, uint32_t arg2);
+DEF_HELPER_FLAGS_2(div_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
+DEF_HELPER_FLAGS_2(rem_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
+DEF_HELPER_FLAGS_2(divu_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(remu_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 
-int64_t tcg_helper_shl_i64(int64_t arg1, int64_t arg2);
-int64_t tcg_helper_shr_i64(int64_t arg1, int64_t arg2);
-int64_t tcg_helper_sar_i64(int64_t arg1, int64_t arg2);
-int64_t tcg_helper_div_i64(int64_t arg1, int64_t arg2);
-int64_t tcg_helper_rem_i64(int64_t arg1, int64_t arg2);
-int64_t tcg_helper_mulsh_i64(int64_t arg1, int64_t arg2);
-uint64_t tcg_helper_divu_i64(uint64_t arg1, uint64_t arg2);
-uint64_t tcg_helper_remu_i64(uint64_t arg1, uint64_t arg2);
-uint64_t tcg_helper_muluh_i64(uint64_t arg1, uint64_t arg2);
+DEF_HELPER_FLAGS_2(shl_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(shr_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(sar_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
 
-#endif
+DEF_HELPER_FLAGS_2(mulsh_i64, TCG_CALL_NO_RWG_SE, s64, s64, s64)
+DEF_HELPER_FLAGS_2(muluh_i64, TCG_CALL_NO_RWG_SE, i64, i64, i64)
diff --git a/tcg/tcg.c b/tcg/tcg.c
index ea8aa70..dae4b7c 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -86,8 +86,14 @@
     uintptr_t func_len;
 } DebugFrameFDEHeader;
 
+typedef struct QEMU_PACKED {
+    DebugFrameCIE cie;
+    DebugFrameFDEHeader fde;
+} DebugFrameHeader;
+
 static void tcg_register_jit_int(void *buf, size_t size,
-                                 void *debug_frame, size_t debug_frame_size)
+                                 const void *debug_frame,
+                                 size_t debug_frame_size)
     __attribute__((unused));
 
 /* Forward declarations for functions declared and used in tcg-target.c. */
@@ -307,32 +313,17 @@
     s->pool_current = NULL;
 }
 
-#include "helper.h"
-
 typedef struct TCGHelperInfo {
     void *func;
     const char *name;
+    unsigned flags;
+    unsigned sizemask;
 } TCGHelperInfo;
 
+#include "exec/helper-proto.h"
+
 static const TCGHelperInfo all_helpers[] = {
-#define GEN_HELPER 2
-#include "helper.h"
-
-    /* Include tcg-runtime.c functions.  */
-    { tcg_helper_div_i32, "div_i32" },
-    { tcg_helper_rem_i32, "rem_i32" },
-    { tcg_helper_divu_i32, "divu_i32" },
-    { tcg_helper_remu_i32, "remu_i32" },
-
-    { tcg_helper_shl_i64, "shl_i64" },
-    { tcg_helper_shr_i64, "shr_i64" },
-    { tcg_helper_sar_i64, "sar_i64" },
-    { tcg_helper_div_i64, "div_i64" },
-    { tcg_helper_rem_i64, "rem_i64" },
-    { tcg_helper_divu_i64, "divu_i64" },
-    { tcg_helper_remu_i64, "remu_i64" },
-    { tcg_helper_mulsh_i64, "mulsh_i64" },
-    { tcg_helper_muluh_i64, "muluh_i64" },
+#include "exec/helper-tcg.h"
 };
 
 void tcg_context_init(TCGContext *s)
@@ -373,7 +364,7 @@
 
     for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
         g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
-                            (gpointer)all_helpers[i].name);
+                            (gpointer)&all_helpers[i]);
     }
 
     tcg_target_init(s);
@@ -706,13 +697,17 @@
 /* Note: we convert the 64 bit args to 32 bit and do some alignment
    and endian swap. Maybe it would be better to do the alignment
    and endian swap in tcg_reg_alloc_call(). */
-void tcg_gen_callN(TCGContext *s, void *func, unsigned int flags,
-                   int sizemask, TCGArg ret, int nargs, TCGArg *args)
+void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret,
+                   int nargs, TCGArg *args)
 {
-    int i;
-    int real_args;
-    int nb_rets;
+    int i, real_args, nb_rets;
+    unsigned sizemask, flags;
     TCGArg *nparam;
+    TCGHelperInfo *info;
+
+    info = g_hash_table_lookup(s->helpers, (gpointer)func);
+    flags = info->flags;
+    sizemask = info->sizemask;
 
 #if defined(__sparc__) && !defined(__arch64__) \
     && !defined(CONFIG_TCG_INTERPRETER)
@@ -798,9 +793,8 @@
     }
     real_args = 0;
     for (i = 0; i < nargs; i++) {
-#if TCG_TARGET_REG_BITS < 64
         int is_64bit = sizemask & (1 << (i+1)*2);
-        if (is_64bit) {
+        if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
             /* some targets want aligned 64 bit args */
             if (real_args & 1) {
@@ -828,7 +822,6 @@
             real_args += 2;
             continue;
         }
-#endif /* TCG_TARGET_REG_BITS < 64 */
 
         *s->gen_opparam_ptr++ = args[i];
         real_args++;
@@ -948,97 +941,26 @@
     return op;
 }
 
-static const TCGOpcode old_ld_opc[8] = {
-    [MO_UB] = INDEX_op_qemu_ld8u,
-    [MO_SB] = INDEX_op_qemu_ld8s,
-    [MO_UW] = INDEX_op_qemu_ld16u,
-    [MO_SW] = INDEX_op_qemu_ld16s,
-#if TCG_TARGET_REG_BITS == 32
-    [MO_UL] = INDEX_op_qemu_ld32,
-    [MO_SL] = INDEX_op_qemu_ld32,
-#else
-    [MO_UL] = INDEX_op_qemu_ld32u,
-    [MO_SL] = INDEX_op_qemu_ld32s,
-#endif
-    [MO_Q]  = INDEX_op_qemu_ld64,
-};
-
-static const TCGOpcode old_st_opc[4] = {
-    [MO_UB] = INDEX_op_qemu_st8,
-    [MO_UW] = INDEX_op_qemu_st16,
-    [MO_UL] = INDEX_op_qemu_st32,
-    [MO_Q]  = INDEX_op_qemu_st64,
-};
-
 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
 {
     memop = tcg_canonicalize_memop(memop, 0, 0);
 
-    if (TCG_TARGET_HAS_new_ldst) {
-        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32;
-        tcg_add_param_i32(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = memop;
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-        return;
-    }
-
-    /* The old opcodes only support target-endian memory operations.  */
-    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
-    assert(old_ld_opc[memop & MO_SSIZE] != 0);
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
-        tcg_add_param_i32(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-    } else {
-        TCGv_i64 val64 = tcg_temp_new_i64();
-
-        *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
-        tcg_add_param_i64(val64);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-
-        tcg_gen_trunc_i64_i32(val, val64);
-        tcg_temp_free_i64(val64);
-    }
+    *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32;
+    tcg_add_param_i32(val);
+    tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = memop;
+    *tcg_ctx.gen_opparam_ptr++ = idx;
 }
 
 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
 {
     memop = tcg_canonicalize_memop(memop, 0, 1);
 
-    if (TCG_TARGET_HAS_new_ldst) {
-        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32;
-        tcg_add_param_i32(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = memop;
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-        return;
-    }
-
-    /* The old opcodes only support target-endian memory operations.  */
-    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
-    assert(old_st_opc[memop & MO_SIZE] != 0);
-
-    if (TCG_TARGET_REG_BITS == 32) {
-        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
-        tcg_add_param_i32(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-    } else {
-        TCGv_i64 val64 = tcg_temp_new_i64();
-
-        tcg_gen_extu_i32_i64(val64, val);
-
-        *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
-        tcg_add_param_i64(val64);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-
-        tcg_temp_free_i64(val64);
-    }
+    *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32;
+    tcg_add_param_i32(val);
+    tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = memop;
+    *tcg_ctx.gen_opparam_ptr++ = idx;
 }
 
 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
@@ -1057,22 +979,10 @@
     }
 #endif
 
-    if (TCG_TARGET_HAS_new_ldst) {
-        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64;
-        tcg_add_param_i64(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = memop;
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-        return;
-    }
-
-    /* The old opcodes only support target-endian memory operations.  */
-    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
-    assert(old_ld_opc[memop & MO_SSIZE] != 0);
-
-    *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
+    *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64;
     tcg_add_param_i64(val);
     tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = memop;
     *tcg_ctx.gen_opparam_ptr++ = idx;
 }
 
@@ -1087,22 +997,10 @@
     }
 #endif
 
-    if (TCG_TARGET_HAS_new_ldst) {
-        *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64;
-        tcg_add_param_i64(val);
-        tcg_add_param_tl(addr);
-        *tcg_ctx.gen_opparam_ptr++ = memop;
-        *tcg_ctx.gen_opparam_ptr++ = idx;
-        return;
-    }
-
-    /* The old opcodes only support target-endian memory operations.  */
-    assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
-    assert(old_st_opc[memop & MO_SIZE] != 0);
-
-    *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
+    *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64;
     tcg_add_param_i64(val);
     tcg_add_param_tl(addr);
+    *tcg_ctx.gen_opparam_ptr++ = memop;
     *tcg_ctx.gen_opparam_ptr++ = idx;
 }
 
@@ -1166,7 +1064,10 @@
 {
     const char *ret = NULL;
     if (s->helpers) {
-        ret = g_hash_table_lookup(s->helpers, (gpointer)val);
+        TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val);
+        if (info) {
+            ret = info->name;
+        }
     }
     return ret;
 }
@@ -2787,7 +2688,8 @@
 }
 
 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
-                                 void *debug_frame, size_t debug_frame_size)
+                                 const void *debug_frame,
+                                 size_t debug_frame_size)
 {
     struct __attribute__((packed)) DebugInfo {
         uint32_t  len;
@@ -2925,10 +2827,10 @@
 
     uintptr_t buf = (uintptr_t)buf_ptr;
     size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
+    DebugFrameHeader *dfh;
 
     img = g_malloc(img_size);
     *img = img_template;
-    memcpy(img + 1, debug_frame, debug_frame_size);
 
     img->phdr.p_vaddr = buf;
     img->phdr.p_paddr = buf;
@@ -2956,6 +2858,11 @@
     img->di.fn_low_pc = buf;
     img->di.fn_high_pc = buf + buf_size;
 
+    dfh = (DebugFrameHeader *)(img + 1);
+    memcpy(dfh, debug_frame, debug_frame_size);
+    dfh->fde.func_start = buf;
+    dfh->fde.func_len = buf_size;
+
 #ifdef DEBUG_JIT
     /* Enable this block to be able to debug the ELF image file creation.
        One can use readelf, objdump, or other inspection utilities.  */
@@ -2983,7 +2890,8 @@
    and implement the internal function we declared earlier.  */
 
 static void tcg_register_jit_int(void *buf, size_t size,
-                                 void *debug_frame, size_t debug_frame_size)
+                                 const void *debug_frame,
+                                 size_t debug_frame_size)
 {
 }
 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index fbc9310..997a704 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -54,8 +54,6 @@
 #error unsupported
 #endif
 
-#include "tcg-runtime.h"
-
 #if TCG_TARGET_NB_REGS <= 32
 typedef uint32_t TCGRegSet;
 #elif TCG_TARGET_NB_REGS <= 64
@@ -725,8 +723,8 @@
 #define tcg_temp_free_ptr(T) tcg_temp_free_i64(TCGV_PTR_TO_NAT(T))
 #endif
 
-void tcg_gen_callN(TCGContext *s, void *func, unsigned int flags,
-                   int sizemask, TCGArg ret, int nargs, TCGArg *args);
+void tcg_gen_callN(TCGContext *s, void *func,
+                   TCGArg ret, int nargs, TCGArg *args);
 
 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
                         int c, int right, int arith);
@@ -911,19 +909,6 @@
 # define helper_ret_stq_mmu   helper_le_stq_mmu
 #endif
 
-uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint16_t helper_ldw_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint32_t helper_ldl_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-uint64_t helper_ldq_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
-
-void helper_stb_mmu(CPUArchState *env, target_ulong addr,
-                    uint8_t val, int mmu_idx);
-void helper_stw_mmu(CPUArchState *env, target_ulong addr,
-                    uint16_t val, int mmu_idx);
-void helper_stl_mmu(CPUArchState *env, target_ulong addr,
-                    uint32_t val, int mmu_idx);
-void helper_stq_mmu(CPUArchState *env, target_ulong addr,
-                    uint64_t val, int mmu_idx);
 #endif /* CONFIG_SOFTMMU */
 
 #endif /* TCG_H */
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index 9b39231..03a7b46 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -227,21 +227,11 @@
 #endif
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-    { INDEX_op_qemu_ld8u, { R, L } },
-    { INDEX_op_qemu_ld8s, { R, L } },
-    { INDEX_op_qemu_ld16u, { R, L } },
-    { INDEX_op_qemu_ld16s, { R, L } },
-    { INDEX_op_qemu_ld32, { R, L } },
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld32u, { R, L } },
-    { INDEX_op_qemu_ld32s, { R, L } },
-#endif
-    { INDEX_op_qemu_ld64, { R64, L } },
+    { INDEX_op_qemu_ld_i32, { R, L } },
+    { INDEX_op_qemu_ld_i64, { R64, L } },
 
-    { INDEX_op_qemu_st8, { R, S } },
-    { INDEX_op_qemu_st16, { R, S } },
-    { INDEX_op_qemu_st32, { R, S } },
-    { INDEX_op_qemu_st64, { R64, S } },
+    { INDEX_op_qemu_st_i32, { R, S } },
+    { INDEX_op_qemu_st_i64, { R64, S } },
 
 #if TCG_TARGET_HAS_ext8s_i32
     { INDEX_op_ext8s_i32, { R, R } },
@@ -544,7 +534,10 @@
 
 static inline void tcg_out_call(TCGContext *s, tcg_insn_unit *arg)
 {
+    uint8_t *old_code_ptr = s->code_ptr;
+    tcg_out_op_t(s, INDEX_op_call);
     tcg_out_ri(s, 1, (uintptr_t)arg);
+    old_code_ptr[1] = s->code_ptr - old_code_ptr;
 }
 
 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
@@ -764,58 +757,52 @@
         tcg_out8(s, args[2]);           /* condition */
         tci_out_label(s, args[3]);
         break;
-    case INDEX_op_qemu_ld8u:
-    case INDEX_op_qemu_ld8s:
-    case INDEX_op_qemu_ld16u:
-    case INDEX_op_qemu_ld16s:
-    case INDEX_op_qemu_ld32:
-#if TCG_TARGET_REG_BITS == 64
-    case INDEX_op_qemu_ld32s:
-    case INDEX_op_qemu_ld32u:
-#endif
+    case INDEX_op_qemu_ld_i32:
         tcg_out_r(s, *args++);
         tcg_out_r(s, *args++);
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        tcg_out_r(s, *args++);
-#endif
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
 #ifdef CONFIG_SOFTMMU
         tcg_out_i(s, *args);
 #endif
         break;
-    case INDEX_op_qemu_ld64:
+    case INDEX_op_qemu_ld_i64:
         tcg_out_r(s, *args++);
-#if TCG_TARGET_REG_BITS == 32
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_r(s, *args++);
+        }
         tcg_out_r(s, *args++);
-#endif
-        tcg_out_r(s, *args++);
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        tcg_out_r(s, *args++);
-#endif
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
 #ifdef CONFIG_SOFTMMU
         tcg_out_i(s, *args);
 #endif
         break;
-    case INDEX_op_qemu_st8:
-    case INDEX_op_qemu_st16:
-    case INDEX_op_qemu_st32:
+    case INDEX_op_qemu_st_i32:
         tcg_out_r(s, *args++);
         tcg_out_r(s, *args++);
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        tcg_out_r(s, *args++);
-#endif
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
 #ifdef CONFIG_SOFTMMU
         tcg_out_i(s, *args);
 #endif
         break;
-    case INDEX_op_qemu_st64:
+    case INDEX_op_qemu_st_i64:
         tcg_out_r(s, *args++);
-#if TCG_TARGET_REG_BITS == 32
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_r(s, *args++);
+        }
         tcg_out_r(s, *args++);
-#endif
-        tcg_out_r(s, *args++);
-#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
-        tcg_out_r(s, *args++);
-#endif
+        if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+            tcg_out_r(s, *args++);
+        }
+        tcg_out_i(s, *args++);
 #ifdef CONFIG_SOFTMMU
         tcg_out_i(s, *args);
 #endif
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index 0be5acd..bd1e974 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -118,8 +118,6 @@
 #define TCG_TARGET_HAS_mulu2_i32        1
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-#define TCG_TARGET_HAS_new_ldst         0
-
 /* Number of registers available.
    For 32 bit hosts, we need more than 8 registers (call arguments). */
 /* #define TCG_TARGET_NB_REGS 8 */
diff --git a/tci.c b/tci.c
index 6523ab8..4711ee4 100644
--- a/tci.c
+++ b/tci.c
@@ -26,6 +26,7 @@
 
 #include "qemu-common.h"
 #include "exec/exec-all.h"           /* MAX_OPC_PARAM_IARGS */
+#include "exec/cpu_ldst.h"
 #include "tcg-op.h"
 
 /* Marker for missing code. */
@@ -116,16 +117,6 @@
     tci_reg[index] = value;
 }
 
-static void tci_write_reg8s(TCGReg index, int8_t value)
-{
-    tci_write_reg(index, value);
-}
-
-static void tci_write_reg16s(TCGReg index, int16_t value)
-{
-    tci_write_reg(index, value);
-}
-
 #if TCG_TARGET_REG_BITS == 64
 static void tci_write_reg32s(TCGReg index, int32_t value)
 {
@@ -138,11 +129,6 @@
     tci_write_reg(index, value);
 }
 
-static void tci_write_reg16(TCGReg index, uint16_t value)
-{
-    tci_write_reg(index, value);
-}
-
 static void tci_write_reg32(TCGReg index, uint32_t value)
 {
     tci_write_reg(index, value);
@@ -433,6 +419,53 @@
     return result;
 }
 
+#ifdef CONFIG_SOFTMMU
+# define mmuidx          tci_read_i(&tb_ptr)
+# define qemu_ld_ub \
+    helper_ret_ldub_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_leuw \
+    helper_le_lduw_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_leul \
+    helper_le_ldul_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_leq \
+    helper_le_ldq_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_beuw \
+    helper_be_lduw_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_beul \
+    helper_be_ldul_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_ld_beq \
+    helper_be_ldq_mmu(env, taddr, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_b(X) \
+    helper_ret_stb_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_lew(X) \
+    helper_le_stw_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_lel(X) \
+    helper_le_stl_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_leq(X) \
+    helper_le_stq_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_bew(X) \
+    helper_be_stw_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_bel(X) \
+    helper_be_stl_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+# define qemu_st_beq(X) \
+    helper_be_stq_mmu(env, taddr, X, mmuidx, (uintptr_t)tb_ptr)
+#else
+# define qemu_ld_ub      ldub_p(g2h(taddr))
+# define qemu_ld_leuw    lduw_le_p(g2h(taddr))
+# define qemu_ld_leul    (uint32_t)ldl_le_p(g2h(taddr))
+# define qemu_ld_leq     ldq_le_p(g2h(taddr))
+# define qemu_ld_beuw    lduw_be_p(g2h(taddr))
+# define qemu_ld_beul    (uint32_t)ldl_be_p(g2h(taddr))
+# define qemu_ld_beq     ldq_be_p(g2h(taddr))
+# define qemu_st_b(X)    stb_p(g2h(taddr), X)
+# define qemu_st_lew(X)  stw_le_p(g2h(taddr), X)
+# define qemu_st_lel(X)  stl_le_p(g2h(taddr), X)
+# define qemu_st_leq(X)  stq_le_p(g2h(taddr), X)
+# define qemu_st_bew(X)  stw_be_p(g2h(taddr), X)
+# define qemu_st_bel(X)  stl_be_p(g2h(taddr), X)
+# define qemu_st_beq(X)  stq_be_p(g2h(taddr), X)
+#endif
+
 /* Interpret pseudo code in tb. */
 uintptr_t tcg_qemu_tb_exec(CPUArchState *env, uint8_t *tb_ptr)
 {
@@ -456,9 +489,6 @@
         tcg_target_ulong label;
         TCGCond condition;
         target_ulong taddr;
-#ifndef CONFIG_SOFTMMU
-        tcg_target_ulong host_addr;
-#endif
         uint8_t tmp8;
         uint16_t tmp16;
         uint32_t tmp32;
@@ -466,6 +496,7 @@
 #if TCG_TARGET_REG_BITS == 32
         uint64_t v64;
 #endif
+        TCGMemOp memop;
 
 #if defined(GETPC)
         tci_tb_ptr = (uintptr_t)tb_ptr;
@@ -1086,145 +1117,145 @@
             assert(tb_ptr == old_code_ptr + op_size);
             tb_ptr += (int32_t)t0;
             continue;
-        case INDEX_op_qemu_ld8u:
+        case INDEX_op_qemu_ld_i32:
             t0 = *tb_ptr++;
             taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp8 = helper_ldb_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp8 = *(uint8_t *)(host_addr + GUEST_BASE);
-#endif
-            tci_write_reg8(t0, tmp8);
+            memop = tci_read_i(&tb_ptr);
+            switch (memop) {
+            case MO_UB:
+                tmp32 = qemu_ld_ub;
+                break;
+            case MO_SB:
+                tmp32 = (int8_t)qemu_ld_ub;
+                break;
+            case MO_LEUW:
+                tmp32 = qemu_ld_leuw;
+                break;
+            case MO_LESW:
+                tmp32 = (int16_t)qemu_ld_leuw;
+                break;
+            case MO_LEUL:
+                tmp32 = qemu_ld_leul;
+                break;
+            case MO_BEUW:
+                tmp32 = qemu_ld_beuw;
+                break;
+            case MO_BESW:
+                tmp32 = (int16_t)qemu_ld_beuw;
+                break;
+            case MO_BEUL:
+                tmp32 = qemu_ld_beul;
+                break;
+            default:
+                tcg_abort();
+            }
+            tci_write_reg(t0, tmp32);
             break;
-        case INDEX_op_qemu_ld8s:
+        case INDEX_op_qemu_ld_i64:
             t0 = *tb_ptr++;
+            if (TCG_TARGET_REG_BITS == 32) {
+                t1 = *tb_ptr++;
+            }
             taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp8 = helper_ldb_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp8 = *(uint8_t *)(host_addr + GUEST_BASE);
-#endif
-            tci_write_reg8s(t0, tmp8);
-            break;
-        case INDEX_op_qemu_ld16u:
-            t0 = *tb_ptr++;
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp16 = helper_ldw_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp16 = tswap16(*(uint16_t *)(host_addr + GUEST_BASE));
-#endif
-            tci_write_reg16(t0, tmp16);
-            break;
-        case INDEX_op_qemu_ld16s:
-            t0 = *tb_ptr++;
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp16 = helper_ldw_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp16 = tswap16(*(uint16_t *)(host_addr + GUEST_BASE));
-#endif
-            tci_write_reg16s(t0, tmp16);
-            break;
-#if TCG_TARGET_REG_BITS == 64
-        case INDEX_op_qemu_ld32u:
-            t0 = *tb_ptr++;
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
-#endif
-            tci_write_reg32(t0, tmp32);
-            break;
-        case INDEX_op_qemu_ld32s:
-            t0 = *tb_ptr++;
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
-#endif
-            tci_write_reg32s(t0, tmp32);
-            break;
-#endif /* TCG_TARGET_REG_BITS == 64 */
-        case INDEX_op_qemu_ld32:
-            t0 = *tb_ptr++;
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp32 = helper_ldl_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp32 = tswap32(*(uint32_t *)(host_addr + GUEST_BASE));
-#endif
-            tci_write_reg32(t0, tmp32);
-            break;
-        case INDEX_op_qemu_ld64:
-            t0 = *tb_ptr++;
-#if TCG_TARGET_REG_BITS == 32
-            t1 = *tb_ptr++;
-#endif
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            tmp64 = helper_ldq_mmu(env, taddr, tci_read_i(&tb_ptr));
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            tmp64 = tswap64(*(uint64_t *)(host_addr + GUEST_BASE));
-#endif
+            memop = tci_read_i(&tb_ptr);
+            switch (memop) {
+            case MO_UB:
+                tmp64 = qemu_ld_ub;
+                break;
+            case MO_SB:
+                tmp64 = (int8_t)qemu_ld_ub;
+                break;
+            case MO_LEUW:
+                tmp64 = qemu_ld_leuw;
+                break;
+            case MO_LESW:
+                tmp64 = (int16_t)qemu_ld_leuw;
+                break;
+            case MO_LEUL:
+                tmp64 = qemu_ld_leul;
+                break;
+            case MO_LESL:
+                tmp64 = (int32_t)qemu_ld_leul;
+                break;
+            case MO_LEQ:
+                tmp64 = qemu_ld_leq;
+                break;
+            case MO_BEUW:
+                tmp64 = qemu_ld_beuw;
+                break;
+            case MO_BESW:
+                tmp64 = (int16_t)qemu_ld_beuw;
+                break;
+            case MO_BEUL:
+                tmp64 = qemu_ld_beul;
+                break;
+            case MO_BESL:
+                tmp64 = (int32_t)qemu_ld_beul;
+                break;
+            case MO_BEQ:
+                tmp64 = qemu_ld_beq;
+                break;
+            default:
+                tcg_abort();
+            }
             tci_write_reg(t0, tmp64);
-#if TCG_TARGET_REG_BITS == 32
-            tci_write_reg(t1, tmp64 >> 32);
-#endif
+            if (TCG_TARGET_REG_BITS == 32) {
+                tci_write_reg(t1, tmp64 >> 32);
+            }
             break;
-        case INDEX_op_qemu_st8:
-            t0 = tci_read_r8(&tb_ptr);
+        case INDEX_op_qemu_st_i32:
+            t0 = tci_read_r(&tb_ptr);
             taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            t2 = tci_read_i(&tb_ptr);
-            helper_stb_mmu(env, taddr, t0, t2);
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            *(uint8_t *)(host_addr + GUEST_BASE) = t0;
-#endif
+            memop = tci_read_i(&tb_ptr);
+            switch (memop) {
+            case MO_UB:
+                qemu_st_b(t0);
+                break;
+            case MO_LEUW:
+                qemu_st_lew(t0);
+                break;
+            case MO_LEUL:
+                qemu_st_lel(t0);
+                break;
+            case MO_BEUW:
+                qemu_st_bew(t0);
+                break;
+            case MO_BEUL:
+                qemu_st_bel(t0);
+                break;
+            default:
+                tcg_abort();
+            }
             break;
-        case INDEX_op_qemu_st16:
-            t0 = tci_read_r16(&tb_ptr);
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            t2 = tci_read_i(&tb_ptr);
-            helper_stw_mmu(env, taddr, t0, t2);
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            *(uint16_t *)(host_addr + GUEST_BASE) = tswap16(t0);
-#endif
-            break;
-        case INDEX_op_qemu_st32:
-            t0 = tci_read_r32(&tb_ptr);
-            taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            t2 = tci_read_i(&tb_ptr);
-            helper_stl_mmu(env, taddr, t0, t2);
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            *(uint32_t *)(host_addr + GUEST_BASE) = tswap32(t0);
-#endif
-            break;
-        case INDEX_op_qemu_st64:
+        case INDEX_op_qemu_st_i64:
             tmp64 = tci_read_r64(&tb_ptr);
             taddr = tci_read_ulong(&tb_ptr);
-#ifdef CONFIG_SOFTMMU
-            t2 = tci_read_i(&tb_ptr);
-            helper_stq_mmu(env, taddr, tmp64, t2);
-#else
-            host_addr = (tcg_target_ulong)taddr;
-            *(uint64_t *)(host_addr + GUEST_BASE) = tswap64(tmp64);
-#endif
+            memop = tci_read_i(&tb_ptr);
+            switch (memop) {
+            case MO_UB:
+                qemu_st_b(tmp64);
+                break;
+            case MO_LEUW:
+                qemu_st_lew(tmp64);
+                break;
+            case MO_LEUL:
+                qemu_st_lel(tmp64);
+                break;
+            case MO_LEQ:
+                qemu_st_leq(tmp64);
+                break;
+            case MO_BEUW:
+                qemu_st_bew(tmp64);
+                break;
+            case MO_BEUL:
+                qemu_st_bel(tmp64);
+                break;
+            case MO_BEQ:
+                qemu_st_beq(tmp64);
+                break;
+            default:
+                tcg_abort();
+            }
             break;
         default:
             TODO();
diff --git a/tests/Makefile b/tests/Makefile
index ba93e8a1..361bb7b 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -60,6 +60,8 @@
 check-unit-y += tests/check-qom-interface$(EXESUF)
 gcov-files-check-qom-interface-y = qom/object.c
 check-unit-$(CONFIG_POSIX) += tests/test-vmstate$(EXESUF)
+check-unit-y += tests/test-qemu-opts$(EXESUF)
+gcov-files-test-qemu-opts-y = qom/test-qemu-opts.c
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
@@ -152,6 +154,8 @@
 check-qtest-i386-y += tests/usb-hcd-ehci-test$(EXESUF)
 gcov-files-i386-y += hw/usb/hcd-ehci.c
 gcov-files-i386-y += hw/usb/hcd-uhci.c
+gcov-files-i386-y += hw/usb/dev-hid.c
+gcov-files-i386-y += hw/usb/dev-storage.c
 check-qtest-x86_64-y = $(check-qtest-i386-y)
 gcov-files-i386-y += i386-softmmu/hw/timer/mc146818rtc.c
 gcov-files-x86_64-y = $(subst i386-softmmu/,x86_64-softmmu/,$(gcov-files-i386-y))
@@ -317,8 +321,9 @@
 tests/es1370-test$(EXESUF): tests/es1370-test.o
 tests/intel-hda-test$(EXESUF): tests/intel-hda-test.o
 tests/ioh3420-test$(EXESUF): tests/ioh3420-test.o
-tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o
+tests/usb-hcd-ehci-test$(EXESUF): tests/usb-hcd-ehci-test.o $(libqos-pc-obj-y)
 tests/qemu-iotests/socket_scm_helper$(EXESUF): tests/qemu-iotests/socket_scm_helper.o
+tests/test-qemu-opts$(EXESUF): tests/test-qemu-opts.o libqemuutil.a libqemustub.a
 
 # QTest rules
 
diff --git a/tests/check-qdict.c b/tests/check-qdict.c
index 2ad0f78..a9296f0 100644
--- a/tests/check-qdict.c
+++ b/tests/check-qdict.c
@@ -444,6 +444,92 @@
     QDECREF(test_dict);
 }
 
+static void qdict_join_test(void)
+{
+    QDict *dict1, *dict2;
+    bool overwrite = false;
+    int i;
+
+    dict1 = qdict_new();
+    dict2 = qdict_new();
+
+
+    /* Test everything once without overwrite and once with */
+    do
+    {
+        /* Test empty dicts */
+        qdict_join(dict1, dict2, overwrite);
+
+        g_assert(qdict_size(dict1) == 0);
+        g_assert(qdict_size(dict2) == 0);
+
+
+        /* First iteration: Test movement */
+        /* Second iteration: Test empty source and non-empty destination */
+        qdict_put(dict2, "foo", qint_from_int(42));
+
+        for (i = 0; i < 2; i++) {
+            qdict_join(dict1, dict2, overwrite);
+
+            g_assert(qdict_size(dict1) == 1);
+            g_assert(qdict_size(dict2) == 0);
+
+            g_assert(qdict_get_int(dict1, "foo") == 42);
+        }
+
+
+        /* Test non-empty source and destination without conflict */
+        qdict_put(dict2, "bar", qint_from_int(23));
+
+        qdict_join(dict1, dict2, overwrite);
+
+        g_assert(qdict_size(dict1) == 2);
+        g_assert(qdict_size(dict2) == 0);
+
+        g_assert(qdict_get_int(dict1, "foo") == 42);
+        g_assert(qdict_get_int(dict1, "bar") == 23);
+
+
+        /* Test conflict */
+        qdict_put(dict2, "foo", qint_from_int(84));
+
+        qdict_join(dict1, dict2, overwrite);
+
+        g_assert(qdict_size(dict1) == 2);
+        g_assert(qdict_size(dict2) == !overwrite);
+
+        g_assert(qdict_get_int(dict1, "foo") == overwrite ? 84 : 42);
+        g_assert(qdict_get_int(dict1, "bar") == 23);
+
+        if (!overwrite) {
+            g_assert(qdict_get_int(dict2, "foo") == 84);
+        }
+
+
+        /* Check the references */
+        g_assert(qdict_get(dict1, "foo")->refcnt == 1);
+        g_assert(qdict_get(dict1, "bar")->refcnt == 1);
+
+        if (!overwrite) {
+            g_assert(qdict_get(dict2, "foo")->refcnt == 1);
+        }
+
+
+        /* Clean up */
+        qdict_del(dict1, "foo");
+        qdict_del(dict1, "bar");
+
+        if (!overwrite) {
+            qdict_del(dict2, "foo");
+        }
+    }
+    while (overwrite ^= true);
+
+
+    QDECREF(dict1);
+    QDECREF(dict2);
+}
+
 /*
  * Errors test-cases
  */
@@ -584,6 +670,7 @@
     g_test_add_func("/public/iterapi", qdict_iterapi_test);
     g_test_add_func("/public/flatten", qdict_flatten_test);
     g_test_add_func("/public/array_split", qdict_array_split_test);
+    g_test_add_func("/public/join", qdict_join_test);
 
     g_test_add_func("/errors/put_exists", qdict_put_exists_test);
     g_test_add_func("/errors/get_not_exists", qdict_get_not_exists_test);
diff --git a/tests/libqos/pci.c b/tests/libqos/pci.c
index 7e0907b..c9a0b91 100644
--- a/tests/libqos/pci.c
+++ b/tests/libqos/pci.c
@@ -103,7 +103,7 @@
 
 void qpci_config_writel(QPCIDevice *dev, uint8_t offset, uint32_t value)
 {
-    dev->bus->config_writew(dev->bus, dev->devfn, offset, value);
+    dev->bus->config_writel(dev->bus, dev->devfn, offset, value);
 }
 
 
diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 8cb61fd..8ce2373 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -35,7 +35,7 @@
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
         qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
         qemu_io('-c', 'write -P 0x1 0 512', backing_img)
-        self.vm = iotests.VM().add_drive(test_img)
+        self.vm = iotests.VM().add_drive("blkdebug::" + test_img)
         self.vm.launch()
 
     def tearDown(self):
diff --git a/tests/qemu-iotests/039 b/tests/qemu-iotests/039
index b9cbe99..27fe4bd 100755
--- a/tests/qemu-iotests/039
+++ b/tests/qemu-iotests/039
@@ -47,6 +47,11 @@
 _default_cache_mode "writethrough"
 _supported_cache_modes "writethrough"
 
+_no_dump_exec()
+{
+    (ulimit -c 0; exec "$@")
+}
+
 size=128M
 
 echo
@@ -67,10 +72,7 @@
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-old_ulimit=$(ulimit -c)
-ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
-ulimit -c "$old_ulimit"
+_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
 
 # The dirty bit must be set
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
@@ -103,10 +105,7 @@
 IMGOPTS="compat=1.1,lazy_refcounts=on"
 _make_test_img $size
 
-old_ulimit=$(ulimit -c)
-ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
-ulimit -c "$old_ulimit"
+_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
 
 # The dirty bit must be set
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
@@ -122,10 +121,7 @@
 IMGOPTS="compat=1.1,lazy_refcounts=off"
 _make_test_img $size
 
-old_ulimit=$(ulimit -c)
-ulimit -c 0 # do not produce a core dump on abort(3)
-$QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" | _filter_qemu_io
-ulimit -c "$old_ulimit"
+_no_dump_exec $QEMU_IO -c "write -P 0x5a 0 512" -c "abort" "$TEST_IMG" 2>&1 | _filter_qemu_io
 
 # The dirty bit must not be set since lazy_refcounts=off
 ./qcow2.py "$TEST_IMG" dump-header | grep incompatible_features
diff --git a/tests/qemu-iotests/039.out b/tests/qemu-iotests/039.out
index fb31ae0..67e7744 100644
--- a/tests/qemu-iotests/039.out
+++ b/tests/qemu-iotests/039.out
@@ -11,6 +11,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./039: Aborted                 ( ulimit -c 0; exec "$@" )
 incompatible_features     0x1
 ERROR cluster 5 refcount=0 reference=1
 ERROR OFLAG_COPIED data cluster: l2_entry=8000000000050000 refcount=0
@@ -42,6 +43,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./039: Aborted                 ( ulimit -c 0; exec "$@" )
 incompatible_features     0x1
 Repairing cluster 5 refcount=0 reference=1
 wrote 512/512 bytes at offset 0
@@ -52,6 +54,7 @@
 Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
 wrote 512/512 bytes at offset 0
 512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+./039: Aborted                 ( ulimit -c 0; exec "$@" )
 incompatible_features     0x0
 No errors were found on the image.
 
diff --git a/tests/qemu-iotests/067.out b/tests/qemu-iotests/067.out
index 8d271cc..7e090b9 100644
--- a/tests/qemu-iotests/067.out
+++ b/tests/qemu-iotests/067.out
@@ -6,7 +6,7 @@
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk -device virtio-blk-pci,drive=disk,id=virtio0
 QMP_VERSION
 {"return": {}}
-{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "disk", "locked": false, "removable": false, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
@@ -24,7 +24,7 @@
 Testing: -drive file=TEST_DIR/t.qcow2,format=qcow2,if=none,id=disk
 QMP_VERSION
 {"return": {}}
-{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
+{"return": [{"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
@@ -44,7 +44,7 @@
 QMP_VERSION
 {"return": {}}
 {"return": "OK\r\n"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
@@ -64,14 +64,14 @@
 QMP_VERSION
 {"return": {}}
 {"return": {}}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"return": {}}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"path": "/machine/peripheral/virtio0/virtio-backend"}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_DELETED", "data": {"device": "virtio0", "path": "/machine/peripheral/virtio0"}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "RESET"}
-{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
+{"return": [{"io-status": "ok", "device": "ide1-cd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "floppy0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"device": "sd0", "locked": false, "removable": true, "tray_open": false, "type": "unknown"}, {"io-status": "ok", "device": "disk", "locked": false, "removable": true, "inserted": {"iops_rd": 0, "detect_zeroes": "off", "image": {"virtual-size": 134217728, "filename": "TEST_DIR/t.qcow2", "cluster-size": 65536, "format": "qcow2", "actual-size": SIZE, "format-specific": {"type": "qcow2", "data": {"compat": "1.1", "lazy-refcounts": false}}, "dirty-flag": false}, "iops_wr": 0, "ro": false, "backing_file_depth": 0, "drv": "qcow2", "iops": 0, "bps_wr": 0, "encrypted": false, "bps": 0, "bps_rd": 0, "file": "TEST_DIR/t.qcow2", "encryption_key_missing": false}, "tray_open": false, "type": "unknown"}]}
 {"return": {}}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "SHUTDOWN"}
 {"timestamp": {"seconds":  TIMESTAMP, "microseconds":  TIMESTAMP}, "event": "DEVICE_TRAY_MOVED", "data": {"device": "ide1-cd0", "tray-open": true}}
diff --git a/tests/qemu-iotests/070 b/tests/qemu-iotests/070
index ce71fa4..ea0dae7 100755
--- a/tests/qemu-iotests/070
+++ b/tests/qemu-iotests/070
@@ -72,6 +72,13 @@
 $QEMU_IO -r -c "read -pP 0xa5 0 18M" "$TEST_IMG" 2>&1 | _filter_testdir \
                                                       | _filter_qemu_io
 
+_cleanup_test_img
+_use_sample_img test-disk2vhd.vhdx.bz2
+
+echo
+echo "=== Verify image created by Disk2VHD can be opened ==="
+$QEMU_IMG info "$TEST_IMG" 2>&1 | _filter_testdir | _filter_qemu
+
 # success, all done
 echo "*** done"
 rm -f $seq.full
diff --git a/tests/qemu-iotests/070.out b/tests/qemu-iotests/070.out
index 922d62c..15f1fc1 100644
--- a/tests/qemu-iotests/070.out
+++ b/tests/qemu-iotests/070.out
@@ -18,4 +18,11 @@
 === Verify open image read-only succeeds after log replay ===
 read 18874368/18874368 bytes at offset 0
 18 MiB, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Verify image created by Disk2VHD can be opened ===
+image: TEST_DIR/test-disk2vhd.vhdx
+file format: vhdx
+virtual size: 256M (268435456 bytes)
+disk size: 260M
+cluster_size: 2097152
 *** done
diff --git a/tests/qemu-iotests/089 b/tests/qemu-iotests/089
new file mode 100755
index 0000000..dffc977
--- /dev/null
+++ b/tests/qemu-iotests/089
@@ -0,0 +1,129 @@
+#!/bin/bash
+#
+# Test case for support of JSON filenames
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=mreitz@redhat.com
+
+seq="$(basename $0)"
+echo "QA output created by $seq"
+
+here="$PWD"
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2
+_supported_proto file
+_supported_os Linux
+
+# Using an image filename containing quotation marks will render the JSON data
+# below invalid. In that case, we have little choice but simply not to run this
+# test.
+case $TEST_IMG in
+    *'"'*)
+        _notrun "image filename may not contain quotation marks"
+        ;;
+esac
+
+IMG_SIZE=64M
+
+# Taken from test 072
+echo
+echo "=== Testing nested image formats ==="
+echo
+
+TEST_IMG="$TEST_IMG.base" _make_test_img $IMG_SIZE
+
+$QEMU_IO -c 'write -P 42 0 512' -c 'write -P 23 512 512' \
+         -c 'write -P 66 1024 512' "$TEST_IMG.base" | _filter_qemu_io
+
+$QEMU_IMG convert -f raw -O $IMGFMT "$TEST_IMG.base" "$TEST_IMG"
+
+$QEMU_IO -c 'read -P 42 0 512' -c 'read -P 23 512 512' \
+         -c 'read -P 66 1024 512' "json:{
+    \"driver\": \"$IMGFMT\",
+    \"file\": {
+        \"driver\": \"$IMGFMT\",
+        \"file\": {
+            \"filename\": \"$TEST_IMG\"
+        }
+    }
+}" | _filter_qemu_io
+
+# This should fail (see test 072)
+$QEMU_IO -c 'read -P 42 0 512' "$TEST_IMG" | _filter_qemu_io
+
+
+# Taken from test 071
+echo
+echo "=== Testing blkdebug ==="
+echo
+
+_make_test_img $IMG_SIZE
+
+$QEMU_IO -c 'write -P 42 0x38000 512' "$TEST_IMG" | _filter_qemu_io
+
+# The "image.filename" part tests whether "a": { "b": "c" } and "a.b": "c" do
+# the same (which they should).
+$QEMU_IO -c 'read -P 42 0x38000 512' "json:{
+    \"driver\": \"$IMGFMT\",
+    \"file\": {
+        \"driver\": \"blkdebug\",
+        \"inject-error\": [{
+            \"event\": \"l2_load\"
+        }],
+        \"image.filename\": \"$TEST_IMG\"
+    }
+}" | _filter_qemu_io
+
+
+echo
+echo "=== Testing qemu-img info output ==="
+echo
+
+TEST_IMG="json:{\"driver\":\"qcow2\",\"file.filename\":\"$TEST_IMG\"}" _img_info
+
+
+echo
+echo "=== Testing option merging ==="
+echo
+
+# Both options given directly and those given in the filename should be used
+$QEMU_IO -c "open -o driver=qcow2 json:{\"file.filename\":\"$TEST_IMG\"}" \
+         -c "info" 2>&1 | _filter_testdir | _filter_imgfmt
+
+# Options given directly should be prioritized over those given in the filename
+$QEMU_IO -c "open -o driver=qcow2 json:{\"driver\":\"raw\",\"file.filename\":\"$TEST_IMG\"}" \
+         -c "info" 2>&1 | _filter_testdir | _filter_imgfmt
+
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/089.out b/tests/qemu-iotests/089.out
new file mode 100644
index 0000000..4ca2f88
--- /dev/null
+++ b/tests/qemu-iotests/089.out
@@ -0,0 +1,50 @@
+QA output created by 089
+
+=== Testing nested image formats ===
+
+Formatting 'TEST_DIR/t.IMGFMT.base', fmt=IMGFMT size=67108864 
+wrote 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+wrote 512/512 bytes at offset 1024
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 512
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read 512/512 bytes at offset 1024
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+Pattern verification failed at offset 0, 512 bytes
+read 512/512 bytes at offset 0
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+
+=== Testing blkdebug ===
+
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+wrote 512/512 bytes at offset 229376
+512 bytes, X ops; XX:XX:XX.X (XXX YYY/sec and XXX ops/sec)
+read failed: Input/output error
+
+=== Testing qemu-img info output ===
+
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 64M (67108864 bytes)
+cluster_size: 65536
+
+=== Testing option merging ===
+
+format name: IMGFMT
+cluster size: 64 KiB
+vm state offset: 512 MiB
+Format specific information:
+    compat: 1.1
+    lazy refcounts: false
+format name: IMGFMT
+cluster size: 64 KiB
+vm state offset: 512 MiB
+Format specific information:
+    compat: 1.1
+    lazy refcounts: false
+*** done
diff --git a/tests/qemu-iotests/091 b/tests/qemu-iotests/091
index 384b3ac..32bbd56 100755
--- a/tests/qemu-iotests/091
+++ b/tests/qemu-iotests/091
@@ -47,6 +47,8 @@
 _supported_fmt qcow2
 _supported_proto file
 _supported_os Linux
+_default_cache_mode "none"
+_supported_cache_modes "writethrough" "none" "writeback"
 
 size=1G
 
@@ -59,13 +61,13 @@
 echo
 
 qemu_comm_method="monitor"
-_launch_qemu -drive file="${TEST_IMG}",cache=none,id=disk
+_launch_qemu -drive file="${TEST_IMG}",cache=${CACHEMODE},id=disk
 h1=$QEMU_HANDLE
 
 echo
 echo === Starting QEMU VM2 ===
 echo
-_launch_qemu -drive file="${TEST_IMG}",cache=none,id=disk \
+_launch_qemu -drive file="${TEST_IMG}",cache=${CACHEMODE},id=disk \
              -incoming "exec: cat '${MIG_FIFO}'"
 h2=$QEMU_HANDLE
 
diff --git a/tests/qemu-iotests/092 b/tests/qemu-iotests/092
new file mode 100755
index 0000000..a8c0c9c
--- /dev/null
+++ b/tests/qemu-iotests/092
@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# qcow1 format input validation tests
+#
+# Copyright (C) 2014 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    rm -f $TEST_IMG.snap
+    _cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow
+_supported_proto generic
+_supported_os Linux
+
+offset_backing_file_offset=8
+offset_backing_file_size=16
+offset_size=24
+offset_cluster_bits=32
+offset_l2_bits=33
+
+echo
+echo "== Invalid cluster size =="
+_make_test_img 64M
+poke_file "$TEST_IMG" "$offset_cluster_bits" "\xff"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_cluster_bits" "\x1f"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_cluster_bits" "\x08"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_cluster_bits" "\x11"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== Invalid L2 table size =="
+_make_test_img 64M
+poke_file "$TEST_IMG" "$offset_l2_bits" "\xff"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_l2_bits" "\x05"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_l2_bits" "\x0e"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+# 1 << 0x1b = 2^31 / L2_CACHE_SIZE
+poke_file "$TEST_IMG" "$offset_l2_bits" "\x1b"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== Invalid size =="
+_make_test_img 64M
+poke_file "$TEST_IMG" "$offset_size" "\xee\xee\xee\xee\xee\xee\xee\xee"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_size" "\x7f\xff\xff\xff\xff\xff\xff\xff"
+{ $QEMU_IO -c "write 0 64M" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+echo
+echo "== Invalid backing file length =="
+_make_test_img 64M
+poke_file "$TEST_IMG" "$offset_backing_file_offset" "\x00\x00\x00\xff"
+poke_file "$TEST_IMG" "$offset_backing_file_size" "\xff\xff\xff\xff"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+poke_file "$TEST_IMG" "$offset_backing_file_size" "\x7f\xff\xff\xff"
+{ $QEMU_IO -c "read 0 512" $TEST_IMG; } 2>&1 | _filter_qemu_io | _filter_testdir
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/092.out b/tests/qemu-iotests/092.out
new file mode 100644
index 0000000..496d8f0
--- /dev/null
+++ b/tests/qemu-iotests/092.out
@@ -0,0 +1,38 @@
+QA output created by 092
+
+== Invalid cluster size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: Cluster size must be between 512 and 64k
+no file open, try 'help open'
+
+== Invalid L2 table size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: L2 table size must be between 512 and 64k
+no file open, try 'help open'
+
+== Invalid size ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: Image too large
+no file open, try 'help open'
+
+== Invalid backing file length ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=67108864 
+qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+no file open, try 'help open'
+qemu-io: can't open device TEST_DIR/t.qcow: Backing file name too long
+no file open, try 'help open'
+*** done
diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter
index 776985d..a04df7f 100644
--- a/tests/qemu-iotests/common.filter
+++ b/tests/qemu-iotests/common.filter
@@ -150,6 +150,7 @@
 _filter_qemu_io()
 {
     _filter_win32 | sed -e "s/[0-9]* ops\; [0-9/:. sec]* ([0-9/.inf]* [EPTGMKiBbytes]*\/sec and [0-9/.inf]* ops\/sec)/X ops\; XX:XX:XX.X (XXX YYY\/sec and XXX ops\/sec)/" \
+        -e "s/: line [0-9][0-9]*:  *[0-9][0-9]*\( Aborted\)/:\1/" \
         -e "s/qemu-io> //g"
 }
 
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index cd3e4d2..0f07440 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -95,5 +95,7 @@
 086 rw auto quick
 087 rw auto
 088 rw auto
+089 rw auto quick
 090 rw auto quick
 091 rw auto
+092 rw auto quick
diff --git a/tests/qemu-iotests/sample_images/test-disk2vhd.vhdx.bz2 b/tests/qemu-iotests/sample_images/test-disk2vhd.vhdx.bz2
new file mode 100644
index 0000000..2891c9a
--- /dev/null
+++ b/tests/qemu-iotests/sample_images/test-disk2vhd.vhdx.bz2
Binary files differ
diff --git a/tests/qom-test.c b/tests/qom-test.c
index 6d9a00b..d8d1d8d 100644
--- a/tests/qom-test.c
+++ b/tests/qom-test.c
@@ -44,7 +44,7 @@
     return false;
 }
 
-static void test_properties(const char *path)
+static void test_properties(const char *path, bool recurse)
 {
     char *child_path;
     QDict *response, *tuple;
@@ -56,14 +56,21 @@
                    "  'arguments': { 'path': '%s' } }", path);
     g_assert(response);
 
+    if (!recurse) {
+        return;
+    }
+
     g_assert(qdict_haskey(response, "return"));
     list = qobject_to_qlist(qdict_get(response, "return"));
     QLIST_FOREACH_ENTRY(list, entry) {
         tuple = qobject_to_qdict(qlist_entry_obj(entry));
-        if (strstart(qdict_get_str(tuple, "type"), "child<", NULL)) {
+        bool is_child = strstart(qdict_get_str(tuple, "type"), "child<", NULL);
+        bool is_link = strstart(qdict_get_str(tuple, "type"), "link<", NULL);
+
+        if (is_child || is_link) {
             child_path = g_strdup_printf("%s/%s",
                                          path, qdict_get_str(tuple, "name"));
-            test_properties(child_path);
+            test_properties(child_path, is_child);
             g_free(child_path);
         } else {
             const char *prop = qdict_get_str(tuple, "name");
@@ -87,7 +94,7 @@
     args = g_strdup_printf("-machine %s", machine);
     qtest_start(args);
 
-    test_properties("/machine");
+    test_properties("/machine", true);
 
     response = qmp("{ 'execute': 'quit' }");
     g_assert(qdict_haskey(response, "return"));
diff --git a/tests/tcg/lm32/Makefile b/tests/tcg/lm32/Makefile
index 8e5d405..57e7363 100644
--- a/tests/tcg/lm32/Makefile
+++ b/tests/tcg/lm32/Makefile
@@ -3,7 +3,7 @@
 CROSS=lm32-elf-
 
 SIM = qemu-system-lm32
-SIMFLAGS = -M lm32-evr -nographic -device lm32-sys -net none -kernel
+SIMFLAGS = -M lm32-evr -nographic -semihosting -net none -kernel
 
 CC      = $(CROSS)gcc
 AS      = $(CROSS)as
@@ -18,6 +18,7 @@
 ASFLAGS += -Wa,-I,$(TSRC_PATH)/
 
 CRT        = crt.o
+HELPER     = helper.o
 TESTCASES += test_add.tst
 TESTCASES += test_addi.tst
 TESTCASES += test_and.tst
@@ -91,15 +92,15 @@
 %.o: $(TSRC_PATH)/%.S
 	$(AS) $(ASFLAGS) -c $< -o $@
 
-%.tst: %.o $(TSRC_PATH)/macros.inc $(CRT)
-	$(LD) $(LDFLAGS) $(NOSTDFLAGS) $(CRT) $< -o $@
+%.tst: %.o $(TSRC_PATH)/macros.inc $(CRT) $(HELPER)
+	$(LD) $(LDFLAGS) $(NOSTDFLAGS) $(CRT) $(HELPER) $< -o $@
 
-build: $(CRT) $(TESTCASES)
+build: $(TESTCASES)
 
 check: $(TESTCASES:test_%.tst=check_%)
 
-check_%: test_%.tst $(CRT) $(SYS)
-	$(SIM) $(SIMFLAGS) $<
+check_%: test_%.tst
+	@$(SIM) $(SIMFLAGS) $<
 
 clean:
-	$(RM) -fr $(TESTCASES) $(CRT)
+	$(RM) -fr $(TESTCASES) $(CRT) $(HELPER)
diff --git a/tests/tcg/lm32/crt.S b/tests/tcg/lm32/crt.S
index 5f9cfd9..fc437a3 100644
--- a/tests/tcg/lm32/crt.S
+++ b/tests/tcg/lm32/crt.S
@@ -8,9 +8,9 @@
 	ori r1, r1, lo(_start)
 	wcsr eba, r1
 	wcsr deba, r1
+	mvhi sp, hi(_fstack)
+	ori sp, sp, lo(_fstack)
 	bi _main
-	nop
-	nop
 
 _breakpoint_handler:
 	ori r25, r25, 1
diff --git a/tests/tcg/lm32/helper.S b/tests/tcg/lm32/helper.S
new file mode 100644
index 0000000..3351d41
--- /dev/null
+++ b/tests/tcg/lm32/helper.S
@@ -0,0 +1,65 @@
+.text
+.global _start, _write, _exit
+.global _tc_fail, _tc_pass
+
+_write:
+	addi sp, sp, -4
+	sw (sp+4), r8
+	mvi r8, 5
+	scall
+	lw r8, (sp+4)
+	addi sp, sp, 4
+	ret
+
+_exit:
+	mvi r8, 1
+	scall
+1:
+	bi 1b
+
+_tc_pass:
+.data
+1:
+	.ascii "OK\n"
+2:
+.text
+	addi sp, sp, -16
+	sw (sp+4), ra
+	sw (sp+8), r1
+	sw (sp+12), r2
+	sw (sp+16), r3
+	mvi r1, 1
+	mvhi r2, hi(1b)
+	ori r2, r2, lo(1b)
+	mvi r3, (2b - 1b)
+	calli _write
+	lw r3, (sp+16)
+	lw r2, (sp+12)
+	lw r1, (sp+8)
+	lw ra, (sp+4)
+	addi sp, sp, 16
+	ret
+
+_tc_fail:
+.data
+1:
+	.ascii "FAILED\n"
+2:
+.text
+	addi sp, sp, -16
+	sw (sp+4), ra
+	sw (sp+8), r1
+	sw (sp+12), r2
+	sw (sp+16), r3
+	sw (sp+4), ra
+	mvi r1, 1
+	mvhi r2, hi(1b)
+	ori r2, r2, lo(1b)
+	mvi r3, (2b - 1b)
+	calli _write
+	lw r3, (sp+16)
+	lw r2, (sp+12)
+	lw r1, (sp+8)
+	lw ra, (sp+4)
+	addi sp, sp, 16
+	ret
diff --git a/tests/tcg/lm32/macros.inc b/tests/tcg/lm32/macros.inc
index 367c7c5..360ad53 100644
--- a/tests/tcg/lm32/macros.inc
+++ b/tests/tcg/lm32/macros.inc
@@ -1,12 +1,26 @@
 
+.equ MAX_TESTNAME_LEN, 32
 .macro test_name name
 	.data
 tn_\name:
-	.asciz "\name"
+	.ascii "\name"
+	.space MAX_TESTNAME_LEN - (. - tn_\name), ' '
 	.text
-	mvhi r13, hi(tn_\name)
-	ori r13, r13, lo(tn_\name)
-	sw (r12+8), r13
+	.global \name
+\name:
+	addi sp, sp, -12
+	sw (sp+4), r1
+	sw (sp+8), r2
+	sw (sp+12), r3
+	mvi r1, 1
+	mvhi r2, hi(tn_\name)
+	ori r2, r2, lo(tn_\name)
+	mvi r3, MAX_TESTNAME_LEN
+	calli _write
+	lw r3, (sp+12)
+	lw r2, (sp+8)
+	lw r1, (sp+4)
+	addi sp, sp, 12
 .endm
 
 .macro load reg val
@@ -15,13 +29,12 @@
 .endm
 
 .macro tc_pass
-	mvi r13, 0
-	sw (r12+4), r13
+	calli _tc_pass
 .endm
 
 .macro tc_fail
-	mvi r13, 1
-	sw (r12+4), r13
+	addi r12, r12, 1
+	calli _tc_fail
 .endm
 
 .macro check_r3 val
@@ -63,14 +76,12 @@
 	.global _main
 	.text
 _main:
-	mvhi r12, hi(0xffff0000)      # base address of test block
-	ori r12, r12, lo(0xffff0000)
+	mvi r12, 0
 .endm
 
 .macro end
-	sw (r12+0), r0
-1:
-	bi 1b
+	mv r1, r12
+	calli _exit
 .endm
 
 # base +
diff --git a/tests/tcg/lm32/test_lb.S b/tests/tcg/lm32/test_lb.S
index f84d21e..d677eea 100644
--- a/tests/tcg/lm32/test_lb.S
+++ b/tests/tcg/lm32/test_lb.S
@@ -8,10 +8,12 @@
 check_r3 0x7e
 
 test_name LB_2
+load r1 data
 lb r3, (r1+1)
 check_r3 0x7f
 
 test_name LB_3
+load r1 data
 lb r3, (r1+-1)
 check_r3 0x7d
 
@@ -21,10 +23,12 @@
 check_r3 0xfffffffe
 
 test_name LB_5
+load r1 data_msb
 lb r3, (r1+1)
 check_r3 0xffffffff
 
 test_name LB_6
+load r1 data_msb
 lb r3, (r1+-1)
 check_r3 0xfffffffd
 
diff --git a/tests/tcg/lm32/test_lbu.S b/tests/tcg/lm32/test_lbu.S
index 4c1786a..dc5d5f6 100644
--- a/tests/tcg/lm32/test_lbu.S
+++ b/tests/tcg/lm32/test_lbu.S
@@ -8,10 +8,12 @@
 check_r3 0x7e
 
 test_name LBU_2
+load r1 data
 lbu r3, (r1+1)
 check_r3 0x7f
 
 test_name LBU_3
+load r1 data
 lbu r3, (r1+-1)
 check_r3 0x7d
 
@@ -21,10 +23,12 @@
 check_r3 0xfe
 
 test_name LBU_5
+load r1 data_msb
 lbu r3, (r1+1)
 check_r3 0xff
 
 test_name LBU_6
+load r1 data_msb
 lbu r3, (r1+-1)
 check_r3 0xfd
 
diff --git a/tests/tcg/lm32/test_lh.S b/tests/tcg/lm32/test_lh.S
index e57d9e3..397996b 100644
--- a/tests/tcg/lm32/test_lh.S
+++ b/tests/tcg/lm32/test_lh.S
@@ -8,10 +8,12 @@
 check_r3 0x7e7f
 
 test_name LH_2
+load r1 data
 lh r3, (r1+2)
 check_r3 0x7071
 
 test_name LH_3
+load r1 data
 lh r3, (r1+-2)
 check_r3 0x7c7d
 
@@ -21,10 +23,12 @@
 check_r3 0xfffffeff
 
 test_name LH_5
+load r1 data_msb
 lh r3, (r1+2)
 check_r3 0xfffff0f1
 
 test_name LH_6
+load r1 data_msb
 lh r3, (r1+-2)
 check_r3 0xfffffcfd
 
diff --git a/tests/tcg/lm32/test_lhu.S b/tests/tcg/lm32/test_lhu.S
index e648775..8de7c52 100644
--- a/tests/tcg/lm32/test_lhu.S
+++ b/tests/tcg/lm32/test_lhu.S
@@ -8,10 +8,12 @@
 check_r3 0x7e7f
 
 test_name LHU_2
+load r1 data
 lhu r3, (r1+2)
 check_r3 0x7071
 
 test_name LHU_3
+load r1 data
 lhu r3, (r1+-2)
 check_r3 0x7c7d
 
@@ -21,10 +23,12 @@
 check_r3 0xfeff
 
 test_name LHU_5
+load r1 data_msb
 lhu r3, (r1+2)
 check_r3 0xf0f1
 
 test_name LHU_6
+load r1 data_msb
 lhu r3, (r1+-2)
 check_r3 0xfcfd
 
diff --git a/tests/tcg/lm32/test_lw.S b/tests/tcg/lm32/test_lw.S
index f8c919d..996e5f8 100644
--- a/tests/tcg/lm32/test_lw.S
+++ b/tests/tcg/lm32/test_lw.S
@@ -8,10 +8,12 @@
 check_r3 0x7e7f7071
 
 test_name LW_2
+load r1 data
 lw r3, (r1+4)
 check_r3 0x72737475
 
 test_name LW_3
+load r1 data
 lw r3, (r1+-4)
 check_r3 0x7a7b7c7d
 
diff --git a/tests/tcg/lm32/test_sb.S b/tests/tcg/lm32/test_sb.S
index 89e39d6..b15a89d 100644
--- a/tests/tcg/lm32/test_sb.S
+++ b/tests/tcg/lm32/test_sb.S
@@ -9,11 +9,13 @@
 check_mem data 0xaa000000
 
 test_name SB_2
+load r1 data
 load r2 0xf0f1f2bb
 sb (r1+1), r2
 check_mem data 0xaabb0000
 
 test_name SB_3
+load r1 data
 load r2 0xf0f1f2cc
 sb (r1+-1), r2
 check_mem data0 0x000000cc
diff --git a/tests/tcg/lm32/test_scall.S b/tests/tcg/lm32/test_scall.S
index b442e32..46032f8 100644
--- a/tests/tcg/lm32/test_scall.S
+++ b/tests/tcg/lm32/test_scall.S
@@ -5,6 +5,10 @@
 test_name SCALL_1
 mvi r1, 1
 wcsr IE, r1
+# we are running in a semi hosted environment
+# therefore we have to set r8 to some unused system
+# call
+mvi r8, 0
 insn:
 scall
 check_excp 64
diff --git a/tests/tcg/lm32/test_sh.S b/tests/tcg/lm32/test_sh.S
index ea8b3f2..bba1022 100644
--- a/tests/tcg/lm32/test_sh.S
+++ b/tests/tcg/lm32/test_sh.S
@@ -9,11 +9,13 @@
 check_mem data 0xaaaa0000
 
 test_name SH_2
+load r1 data
 load r2 0xf0f1bbbb
 sh (r1+2), r2
 check_mem data 0xaaaabbbb
 
 test_name SH_3
+load r1 data
 load r2 0xf0f1cccc
 sh (r1+-2), r2
 check_mem data0 0x0000cccc
diff --git a/tests/tcg/lm32/test_sw.S b/tests/tcg/lm32/test_sw.S
index d1fdadc..2b1c017 100644
--- a/tests/tcg/lm32/test_sw.S
+++ b/tests/tcg/lm32/test_sw.S
@@ -9,16 +9,19 @@
 check_mem data 0xaabbccdd
 
 test_name SW_2
+load r1 data
 load r2 0x00112233
 sw (r1+4), r2
 check_mem data1 0x00112233
 
 test_name SW_3
+load r1 data
 load r2 0x44556677
 sw (r1+-4), r2
 check_mem data0 0x44556677
 
 test_name SW_4
+load r1 data
 sw (r1+0), r1
 lw r3, (r1+0)
 check_r3 data
diff --git a/tests/tcg/xtensa/test_mmu.S b/tests/tcg/xtensa/test_mmu.S
index 099031f..58c5bca 100644
--- a/tests/tcg/xtensa/test_mmu.S
+++ b/tests/tcg/xtensa/test_mmu.S
@@ -4,17 +4,29 @@
 
 .purgem test_init
 
+.macro clean_tlb_way way, page_size, n_entries
+    movi    a2, \way
+    movi    a3, \page_size
+    movi    a4, \n_entries
+    loop    a4, 1f
+    idtlb   a2
+    iitlb   a2
+    add     a2, a2, a3
+1:
+.endm
+
 .macro test_init
-    movi    a2, 0x00000004
-    idtlb   a2
-    movi    a2, 0x00100004
-    idtlb   a2
-    movi    a2, 0x00200004
-    idtlb   a2
-    movi    a2, 0x00300004
-    idtlb   a2
+    clean_tlb_way 0, 0x00001000, 4
+    clean_tlb_way 1, 0x00001000, 4
+    clean_tlb_way 2, 0x00001000, 4
+    clean_tlb_way 3, 0x00001000, 4
+    clean_tlb_way 4, 0x00100000, 4
     movi    a2, 0x00000007
     idtlb   a2
+    movi    a2, 0x00000008
+    idtlb   a2
+    movi    a2, 0x00000009
+    idtlb   a2
 .endm
 
 test tlb_group
@@ -508,4 +520,224 @@
     assert_sr exccause, 24
 test_end
 
+test cross_page_insn
+    set_vector kernel, 2f
+
+    movi    a2, 0x04000003 /* PPN */
+    movi    a3, 0x00007000 /* VPN */
+    witlb   a2, a3
+    wdtlb   a2, a3
+    movi    a3, 0x00008000 /* VPN */
+    witlb   a2, a3
+    wdtlb   a2, a3
+
+    movi    a2, 0x00007fff
+    movi    a3, 20f
+    movi    a4, 21f
+    sub     a4, a4, a3
+    loop    a4, 1f
+    l8ui    a5, a3, 0
+    s8i     a5, a2, 0
+    addi    a2, a2, 1
+    addi    a3, a3, 1
+1:
+    movi    a2, 0x00007fff
+    movi    a3, 0x00008000
+    /* DTLB: OK, ITLB: OK */
+    jx      a2
+
+    .begin  no-transform
+20:
+    l32i    a2, a3, 0
+    syscall
+21:
+    .end    no-transform
+
+2:
+    rsr     a2, exccause
+    movi    a3, 1
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x8002
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007fff
+    assert  ne, a2, a3
+
+    reset_ps
+    set_vector kernel, 3f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007fff
+    movi    a3, 0x00008000
+    /* DTLB: FAIL, ITLB: OK */
+    jx      a2
+3:
+    rsr     a2, exccause
+    movi    a3, 28
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x7fff
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007fff
+    assert  eq, a2, a3
+
+    reset_ps
+    set_vector kernel, 4f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    witlb   a2, a3
+    movi    a2, 0x04000003 /* PPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007fff
+    movi    a3, 0x00008000
+    /* DTLB: OK, ITLB: FAIL */
+    jx      a2
+4:
+    rsr     a2, exccause
+    movi    a3, 20
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x7fff
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007fff
+    assert  eq, a2, a3
+
+    reset_ps
+    set_vector kernel, 5f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007fff
+    movi    a3, 0x00008000
+    /* DTLB: FAIL, ITLB: FAIL */
+    jx      a2
+5:
+    rsr     a2, exccause
+    movi    a3, 20
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x7fff
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007fff
+    assert  eq, a2, a3
+test_end
+
+test cross_page_tb
+    set_vector kernel, 2f
+
+    movi    a2, 0x04000003 /* PPN */
+    movi    a3, 0x00007000 /* VPN */
+    witlb   a2, a3
+    wdtlb   a2, a3
+    movi    a3, 0x00008000 /* VPN */
+    witlb   a2, a3
+    wdtlb   a2, a3
+
+    movi    a2, 0x00007ffd
+    movi    a3, 20f
+    movi    a4, 21f
+    sub     a4, a4, a3
+    loop    a4, 1f
+    l8ui    a5, a3, 0
+    s8i     a5, a2, 0
+    addi    a2, a2, 1
+    addi    a3, a3, 1
+1:
+    movi    a2, 0x00007ffd
+    movi    a3, 0x00008000
+    /* DTLB: OK, ITLB: OK */
+    jx      a2
+
+    .begin  no-transform
+20:
+    l32i    a2, a3, 0
+    syscall
+21:
+    .end    no-transform
+
+2:
+    rsr     a2, exccause
+    movi    a3, 1
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x8000
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007ffd
+    assert  ne, a2, a3
+
+    reset_ps
+    set_vector kernel, 3f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007ffd
+    movi    a3, 0x00008000
+    /* DTLB: FAIL, ITLB: OK */
+    jx      a2
+3:
+    rsr     a2, exccause
+    movi    a3, 28
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x7ffd
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007ffd
+    assert  eq, a2, a3
+
+    reset_ps
+    set_vector kernel, 4f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    witlb   a2, a3
+    movi    a2, 0x04000003 /* PPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007ffd
+    movi    a3, 0x00008000
+    /* DTLB: OK, ITLB: FAIL */
+    jx      a2
+4:
+    rsr     a2, exccause
+    movi    a3, 20
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x8000
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007ffd
+    assert  ne, a2, a3
+
+    reset_ps
+    set_vector kernel, 5f
+
+    movi    a2, 0x0400000c /* PPN */
+    movi    a3, 0x00008000 /* VPN */
+    wdtlb   a2, a3
+    movi    a2, 0x00007ffd
+    movi    a3, 0x00008000
+    /* DTLB: FAIL, ITLB: FAIL */
+    jx      a2
+5:
+    rsr     a2, exccause
+    movi    a3, 28
+    assert  eq, a2, a3
+    rsr     a2, epc1
+    movi    a3, 0x7ffd
+    assert  eq, a2, a3
+    rsr     a2, excsave1
+    movi    a3, 0x00007ffd
+    assert  eq, a2, a3
+test_end
+
 test_suite_end
diff --git a/tests/test-qemu-opts.c b/tests/test-qemu-opts.c
new file mode 100644
index 0000000..3653507
--- /dev/null
+++ b/tests/test-qemu-opts.c
@@ -0,0 +1,441 @@
+/*
+ * QemuOpts unit-tests.
+ *
+ * Copyright (C) 2014 Leandro Dorileo <l@dorileo.org>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2.1 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include "qapi/error.h"
+#include "qapi/qmp/qstring.h"
+#include "qemu/config-file.h"
+
+#include <glib.h>
+#include <string.h>
+
+static QemuOptsList opts_list_01 = {
+    .name = "opts_list_01",
+    .head = QTAILQ_HEAD_INITIALIZER(opts_list_01.head),
+    .desc = {
+        {
+            .name = "str1",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "str2",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "str3",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "number1",
+            .type = QEMU_OPT_NUMBER,
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList opts_list_02 = {
+    .name = "opts_list_02",
+    .head = QTAILQ_HEAD_INITIALIZER(opts_list_02.head),
+    .desc = {
+        {
+            .name = "str1",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "bool1",
+            .type = QEMU_OPT_BOOL,
+        },{
+            .name = "str2",
+            .type = QEMU_OPT_STRING,
+        },{
+            .name = "size1",
+            .type = QEMU_OPT_SIZE,
+        },
+        { /* end of list */ }
+    },
+};
+
+QemuOptsList opts_list_03 = {
+    .name = "opts_list_03",
+    .head = QTAILQ_HEAD_INITIALIZER(opts_list_03.head),
+    .desc = {
+        /* no elements => accept any params */
+        { /* end of list */ }
+    },
+};
+
+static void register_opts(void)
+{
+    qemu_add_opts(&opts_list_01);
+    qemu_add_opts(&opts_list_02);
+    qemu_add_opts(&opts_list_03);
+}
+
+static void test_find_unknown_opts(void)
+{
+    QemuOptsList *list;
+    Error *err = NULL;
+
+    /* should not return anything, we don't have an "unknown" option */
+    list = qemu_find_opts_err("unknown", &err);
+    g_assert(list == NULL);
+    g_assert(err);
+    error_free(err);
+}
+
+static void test_qemu_find_opts(void)
+{
+    QemuOptsList *list;
+
+    /* we have an "opts_list_01" option, should return it */
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+}
+
+static void test_qemu_opts_create(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* now we've create the opts, must find it */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts != NULL);
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opt_get(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    const char *opt = NULL;
+
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* haven't set anything to str2 yet */
+    opt = qemu_opt_get(opts, "str2");
+    g_assert(opt == NULL);
+
+    qemu_opt_set(opts, "str2", "value");
+
+    /* now we have set str2, should know about it */
+    opt = qemu_opt_get(opts, "str2");
+    g_assert_cmpstr(opt, ==, "value");
+
+    qemu_opt_set(opts, "str2", "value2");
+
+    /* having reset the value, the returned should be the reset one */
+    opt = qemu_opt_get(opts, "str2");
+    g_assert_cmpstr(opt, ==, "value2");
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opt_get_bool(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    bool opt;
+    int ret;
+
+    list = qemu_find_opts("opts_list_02");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_02");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* haven't set anything to bool1 yet, so defval should be returned */
+    opt = qemu_opt_get_bool(opts, "bool1", false);
+    g_assert(opt == false);
+
+    ret = qemu_opt_set_bool(opts, "bool1", true);
+    g_assert(ret == 0);
+
+    /* now we have set bool1, should know about it */
+    opt = qemu_opt_get_bool(opts, "bool1", false);
+    g_assert(opt == true);
+
+    /* having reset the value, opt should be the reset one not defval */
+    ret = qemu_opt_set_bool(opts, "bool1", false);
+    g_assert(ret == 0);
+
+    opt = qemu_opt_get_bool(opts, "bool1", true);
+    g_assert(opt == false);
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opt_get_number(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    uint64_t opt;
+    int ret;
+
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* haven't set anything to number1 yet, so defval should be returned */
+    opt = qemu_opt_get_number(opts, "number1", 5);
+    g_assert(opt == 5);
+
+    ret = qemu_opt_set_number(opts, "number1", 10);
+    g_assert(ret == 0);
+
+    /* now we have set number1, should know about it */
+    opt = qemu_opt_get_number(opts, "number1", 5);
+    g_assert(opt == 10);
+
+    /* having reset it, the returned should be the reset one not defval */
+    ret = qemu_opt_set_number(opts, "number1", 15);
+    g_assert(ret == 0);
+
+    opt = qemu_opt_get_number(opts, "number1", 5);
+    g_assert(opt == 15);
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opt_get_size(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    uint64_t opt;
+    QDict *dict;
+
+    list = qemu_find_opts("opts_list_02");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_02");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* haven't set anything to size1 yet, so defval should be returned */
+    opt = qemu_opt_get_size(opts, "size1", 5);
+    g_assert(opt == 5);
+
+    dict = qdict_new();
+    g_assert(dict != NULL);
+
+    qdict_put(dict, "size1", qstring_from_str("10"));
+
+    qemu_opts_absorb_qdict(opts, dict, &error_abort);
+    g_assert(error_abort == NULL);
+
+    /* now we have set size1, should know about it */
+    opt = qemu_opt_get_size(opts, "size1", 5);
+    g_assert(opt == 10);
+
+    /* reset value */
+    qdict_put(dict, "size1", qstring_from_str("15"));
+
+    qemu_opts_absorb_qdict(opts, dict, &error_abort);
+    g_assert(error_abort == NULL);
+
+    /* test the reset value */
+    opt = qemu_opt_get_size(opts, "size1", 5);
+    g_assert(opt == 15);
+
+    qdict_del(dict, "size1");
+    g_free(dict);
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opt_unset(void)
+{
+    QemuOpts *opts;
+    const char *value;
+    int ret;
+
+    /* dynamically initialized (parsed) opts */
+    opts = qemu_opts_parse(&opts_list_03, "key=value", 0);
+    g_assert(opts != NULL);
+
+    /* check default/parsed value */
+    value = qemu_opt_get(opts, "key");
+    g_assert_cmpstr(value, ==, "value");
+
+    /* reset it to value2 */
+    qemu_opt_set(opts, "key", "value2");
+
+    value = qemu_opt_get(opts, "key");
+    g_assert_cmpstr(value, ==, "value2");
+
+    /* unset, valid only for "accept any" */
+    ret = qemu_opt_unset(opts, "key");
+    g_assert(ret == 0);
+
+    /* after reset the value should be the parsed/default one */
+    value = qemu_opt_get(opts, "key");
+    g_assert_cmpstr(value, ==, "value");
+
+    qemu_opts_del(opts);
+}
+
+static void test_qemu_opts_reset(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    uint64_t opt;
+    int ret;
+
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* create the opts */
+    opts = qemu_opts_create(list, NULL, 0, &error_abort);
+    g_assert(opts != NULL);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* haven't set anything to number1 yet, so defval should be returned */
+    opt = qemu_opt_get_number(opts, "number1", 5);
+    g_assert(opt == 5);
+
+    ret = qemu_opt_set_number(opts, "number1", 10);
+    g_assert(ret == 0);
+
+    /* now we have set number1, should know about it */
+    opt = qemu_opt_get_number(opts, "number1", 5);
+    g_assert(opt == 10);
+
+    qemu_opts_reset(list);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+static void test_qemu_opts_set(void)
+{
+    QemuOptsList *list;
+    QemuOpts *opts;
+    int ret;
+    const char *opt;
+
+    list = qemu_find_opts("opts_list_01");
+    g_assert(list != NULL);
+    g_assert(QTAILQ_EMPTY(&list->head));
+    g_assert_cmpstr(list->name, ==, "opts_list_01");
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+
+    /* implicitly create opts and set str3 value */
+    ret = qemu_opts_set(list, NULL, "str3", "value");
+    g_assert(ret == 0);
+    g_assert(!QTAILQ_EMPTY(&list->head));
+
+    /* get the just created opts */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts != NULL);
+
+    /* check the str3 value */
+    opt = qemu_opt_get(opts, "str3");
+    g_assert_cmpstr(opt, ==, "value");
+
+    qemu_opts_del(opts);
+
+    /* should not find anything at this point */
+    opts = qemu_opts_find(list, NULL);
+    g_assert(opts == NULL);
+}
+
+int main(int argc, char *argv[])
+{
+    register_opts();
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/qemu-opts/find_unknown_opts", test_find_unknown_opts);
+    g_test_add_func("/qemu-opts/find_opts", test_qemu_find_opts);
+    g_test_add_func("/qemu-opts/opts_create", test_qemu_opts_create);
+    g_test_add_func("/qemu-opts/opt_get", test_qemu_opt_get);
+    g_test_add_func("/qemu-opts/opt_get_bool", test_qemu_opt_get_bool);
+    g_test_add_func("/qemu-opts/opt_get_number", test_qemu_opt_get_number);
+    g_test_add_func("/qemu-opts/opt_get_size", test_qemu_opt_get_size);
+    g_test_add_func("/qemu-opts/opt_unset", test_qemu_opt_unset);
+    g_test_add_func("/qemu-opts/opts_reset", test_qemu_opts_reset);
+    g_test_add_func("/qemu-opts/opts_set", test_qemu_opts_set);
+    g_test_run();
+    return 0;
+}
diff --git a/tests/test-qmp-output-visitor.c b/tests/test-qmp-output-visitor.c
index 9c15458..74020de 100644
--- a/tests/test-qmp-output-visitor.c
+++ b/tests/test-qmp-output-visitor.c
@@ -507,6 +507,15 @@
     qapi_free_UserDefAnonUnion(tmp);
 }
 
+static void test_visitor_out_empty(TestOutputVisitorData *data,
+                                   const void *unused)
+{
+    QObject *arg;
+
+    arg = qmp_output_get_qobject(data->qov);
+    g_assert(!arg);
+}
+
 static void init_native_list(UserDefNativeListUnion *cvalue)
 {
     int i;
@@ -859,6 +868,8 @@
                             &out_visitor_data, test_visitor_out_union_flat);
     output_visitor_test_add("/visitor/output/union-anon",
                             &out_visitor_data, test_visitor_out_union_anon);
+    output_visitor_test_add("/visitor/output/empty",
+                            &out_visitor_data, test_visitor_out_empty);
     output_visitor_test_add("/visitor/output/native_list/int",
                             &out_visitor_data, test_visitor_out_native_list_int);
     output_visitor_test_add("/visitor/output/native_list/int8",
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
index c1f8e13..aa156bc 100644
--- a/tests/test-thread-pool.c
+++ b/tests/test-thread-pool.c
@@ -180,7 +180,7 @@
 
     /* Canceling the others will be a blocking operation.  */
     for (i = 0; i < 100; i++) {
-        if (data[i].n != 3) {
+        if (data[i].aiocb && data[i].n != 3) {
             bdrv_aio_cancel(data[i].aiocb);
         }
     }
diff --git a/tests/usb-hcd-ehci-test.c b/tests/usb-hcd-ehci-test.c
index bc56ba7..bcdf62f 100644
--- a/tests/usb-hcd-ehci-test.c
+++ b/tests/usb-hcd-ehci-test.c
@@ -9,12 +9,149 @@
 
 #include <glib.h>
 #include <string.h>
+#include <stdio.h>
 #include "libqtest.h"
+#include "libqos/pci-pc.h"
 #include "qemu/osdep.h"
+#include "hw/usb/uhci-regs.h"
+#include "hw/usb/ehci-regs.h"
 
-/* Tests only initialization so far. TODO: Replace with functional tests */
-static void pci_nop(void)
+struct qhc {
+    QPCIDevice *dev;
+    void *base;
+};
+
+static QPCIBus *pcibus;
+static struct qhc uhci1;
+static struct qhc uhci2;
+static struct qhc uhci3;
+static struct qhc ehci1;
+
+/* helpers */
+
+static void pci_init_one(struct qhc *hc, uint32_t devfn, int bar)
 {
+    hc->dev = qpci_device_find(pcibus, devfn);
+    g_assert(hc->dev != NULL);
+    qpci_device_enable(hc->dev);
+    hc->base = qpci_iomap(hc->dev, bar);
+    g_assert(hc->base != NULL);
+}
+
+#if 0
+static void uhci_port_update(struct qhc *hc, int port,
+                             uint16_t set, uint16_t clear)
+{
+    void *addr = hc->base + 0x10 + 2 * port;
+    uint16_t value;
+
+    value = qpci_io_readw(hc->dev, addr);
+    value |= set;
+    value &= ~clear;
+    qpci_io_writew(hc->dev, addr, value);
+}
+#endif
+
+static void uhci_port_test(struct qhc *hc, int port, uint16_t expect)
+{
+    void *addr = hc->base + 0x10 + 2 * port;
+    uint16_t value = qpci_io_readw(hc->dev, addr);
+    uint16_t mask = ~(UHCI_PORT_WRITE_CLEAR | UHCI_PORT_RSVD1);
+
+#if 0
+    fprintf(stderr, "%s: %d, have 0x%04x, want 0x%04x\n",
+            __func__, port, value & mask, expect & mask);
+#endif
+    g_assert((value & mask) == (expect & mask));
+}
+
+static void ehci_port_test(struct qhc *hc, int port, uint32_t expect)
+{
+    void *addr = hc->base + 0x64 + 4 * port;
+    uint32_t value = qpci_io_readl(hc->dev, addr);
+    uint16_t mask = ~(PORTSC_CSC | PORTSC_PEDC | PORTSC_OCC);
+
+#if 0
+    fprintf(stderr, "%s: %d, have 0x%08x, want 0x%08x\n",
+            __func__, port, value & mask, expect & mask);
+#endif
+    g_assert((value & mask) == (expect & mask));
+}
+
+/* tests */
+
+static void pci_init(void)
+{
+    if (pcibus) {
+        return;
+    }
+    pcibus = qpci_init_pc();
+    g_assert(pcibus != NULL);
+
+    pci_init_one(&uhci1, QPCI_DEVFN(0x1d, 0), 4);
+    pci_init_one(&uhci2, QPCI_DEVFN(0x1d, 1), 4);
+    pci_init_one(&uhci3, QPCI_DEVFN(0x1d, 2), 4);
+    pci_init_one(&ehci1, QPCI_DEVFN(0x1d, 7), 0);
+}
+
+static void pci_uhci_port_1(void)
+{
+    g_assert(pcibus != NULL);
+
+    uhci_port_test(&uhci1, 0, UHCI_PORT_CCS); /* usb-tablet  */
+    uhci_port_test(&uhci1, 1, UHCI_PORT_CCS); /* usb-storage */
+    uhci_port_test(&uhci2, 0, 0);
+    uhci_port_test(&uhci2, 1, 0);
+    uhci_port_test(&uhci3, 0, 0);
+    uhci_port_test(&uhci3, 1, 0);
+}
+
+static void pci_ehci_port_1(void)
+{
+    int i;
+
+    g_assert(pcibus != NULL);
+
+    for (i = 0; i < 6; i++) {
+        ehci_port_test(&ehci1, i, PORTSC_POWNER | PORTSC_PPOWER);
+    }
+}
+
+static void pci_ehci_config(void)
+{
+    /* hands over all ports from companion uhci to ehci */
+    qpci_io_writew(ehci1.dev, ehci1.base + 0x60, 1);
+}
+
+static void pci_uhci_port_2(void)
+{
+    g_assert(pcibus != NULL);
+
+    uhci_port_test(&uhci1, 0, 0); /* usb-tablet,  @ehci */
+    uhci_port_test(&uhci1, 1, 0); /* usb-storage, @ehci */
+    uhci_port_test(&uhci2, 0, 0);
+    uhci_port_test(&uhci2, 1, 0);
+    uhci_port_test(&uhci3, 0, 0);
+    uhci_port_test(&uhci3, 1, 0);
+}
+
+static void pci_ehci_port_2(void)
+{
+    static uint32_t expect[] = {
+        PORTSC_PPOWER | PORTSC_CONNECT, /* usb-tablet  */
+        PORTSC_PPOWER | PORTSC_CONNECT, /* usb-storage */
+        PORTSC_PPOWER,
+        PORTSC_PPOWER,
+        PORTSC_PPOWER,
+        PORTSC_PPOWER,
+    };
+    int i;
+
+    g_assert(pcibus != NULL);
+
+    for (i = 0; i < 6; i++) {
+        ehci_port_test(&ehci1, i, expect[i]);
+    }
 }
 
 int main(int argc, char **argv)
@@ -22,7 +159,12 @@
     int ret;
 
     g_test_init(&argc, &argv, NULL);
-    qtest_add_func("/ehci/pci/nop", pci_nop);
+    qtest_add_func("/ehci/pci/init", pci_init);
+    qtest_add_func("/ehci/pci/uhci-port-1", pci_uhci_port_1);
+    qtest_add_func("/ehci/pci/ehci-port-1", pci_ehci_port_1);
+    qtest_add_func("/ehci/pci/ehci-config", pci_ehci_config);
+    qtest_add_func("/ehci/pci/uhci-port-2", pci_uhci_port_2);
+    qtest_add_func("/ehci/pci/ehci-port-2", pci_ehci_port_2);
 
     qtest_start("-machine q35 -device ich9-usb-ehci1,bus=pcie.0,addr=1d.7,"
                 "multifunction=on,id=ich9-ehci-1 "
@@ -31,7 +173,10 @@
                 "-device ich9-usb-uhci2,bus=pcie.0,addr=1d.1,"
                 "multifunction=on,masterbus=ich9-ehci-1.0,firstport=2 "
                 "-device ich9-usb-uhci3,bus=pcie.0,addr=1d.2,"
-                "multifunction=on,masterbus=ich9-ehci-1.0,firstport=4");
+                "multifunction=on,masterbus=ich9-ehci-1.0,firstport=4 "
+                "-drive if=none,id=usbcdrom,media=cdrom "
+                "-device usb-tablet,bus=ich9-ehci-1.0,port=1,usb_version=1 "
+                "-device usb-storage,bus=ich9-ehci-1.0,port=2,drive=usbcdrom ");
     ret = g_test_run();
 
     qtest_end();
diff --git a/thread-pool.c b/thread-pool.c
index fbdd3ff..dfb699d 100644
--- a/thread-pool.c
+++ b/thread-pool.c
@@ -224,6 +224,7 @@
         pool->pending_cancellations--;
     }
     qemu_mutex_unlock(&pool->lock);
+    event_notifier_ready(&pool->notifier);
 }
 
 static const AIOCBInfo thread_pool_aiocb_info = {
diff --git a/trace-events b/trace-events
index b6d289d..ffe6e62 100644
--- a/trace-events
+++ b/trace-events
@@ -371,7 +371,7 @@
 usb_xhci_irq_msix_unuse(uint32_t nr) "nr %d"
 usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char *trb, const char *evt, uint64_t param, uint32_t status, uint32_t control) "v %d, idx %d, %s, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
 usb_xhci_fetch_trb(uint64_t addr, const char *name, uint64_t param, uint32_t status, uint32_t control) "addr %016" PRIx64 ", %s, p %016" PRIx64 ", s %08x, c 0x%08x"
-usb_xhci_port_reset(uint32_t port) "port %d"
+usb_xhci_port_reset(uint32_t port, bool warm) "port %d, warm %d"
 usb_xhci_port_link(uint32_t port, uint32_t pls) "port %d, pls %d"
 usb_xhci_port_notify(uint32_t port, uint32_t pls) "port %d, bits %x"
 usb_xhci_slot_enable(uint32_t slotid) "slotid %d"
@@ -627,9 +627,6 @@
 lm32_uart_memory_read(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
 lm32_uart_irq_state(int level) "irq state %d"
 
-# hw/misc/lm32_sys.c
-lm32_sys_memory_write(uint32_t addr, uint32_t value) "addr 0x%08x value 0x%08x"
-
 # hw/scsi/megasas.c
 megasas_init_firmware(uint64_t pa) "pa %" PRIx64 " "
 megasas_init_queue(uint64_t queue_pa, int queue_len, uint64_t head, uint64_t tail, uint32_t flags) "queue at %" PRIx64 " len %d head %" PRIx64 " tail %" PRIx64 " flags %x"
@@ -1045,12 +1042,21 @@
 ppm_save(const char *filename, void *display_surface) "%s surface=%p"
 
 # ui/gtk.c
-gd_switch(int width, int height) "width=%d, height=%d"
-gd_update(int x, int y, int w, int h) "x=%d, y=%d, w=%d, h=%d"
-gd_key_event(int gdk_keycode, int qemu_keycode, const char *action) "translated GDK keycode %d to QEMU keycode %d (%s)"
+gd_switch(const char *tab, int width, int height) "tab=%s, width=%d, height=%d"
+gd_update(const char *tab, int x, int y, int w, int h) "tab=%s, x=%d, y=%d, w=%d, h=%d"
+gd_key_event(const char *tab, int gdk_keycode, int qemu_keycode, const char *action) "tab=%s, translated GDK keycode %d to QEMU keycode %d (%s)"
+gd_grab(const char *tab, const char *device, bool on) "tab=%s, %s %d"
+
+# ui/vnc.c
+vnc_key_guest_leds(bool caps, bool num, bool scroll) "caps %d, num %d, scroll %d"
+vnc_key_map_init(const char *layout) "%s"
+vnc_key_event_ext(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x, keycode 0x%x [%s]"
+vnc_key_event_map(bool down, int sym, int keycode, const char *name) "down %d, sym 0x%x -> keycode 0x%x [%s]"
+vnc_key_sync_numlock(bool on) "%d"
+vnc_key_sync_capslock(bool on) "%d"
 
 # ui/input.c
-input_event_key_number(int conidx, int number, bool down) "con %d, key number 0x%x, down %d"
+input_event_key_number(int conidx, int number, const char *qcode, bool down) "con %d, key number 0x%x [%s], down %d"
 input_event_key_qcode(int conidx, const char *qcode, bool down) "con %d, key qcode %s, down %d"
 input_event_btn(int conidx, const char *btn, bool down) "con %d, button %s, down %d"
 input_event_rel(int conidx, const char *axis, int value) "con %d, axis %s, value %d"
diff --git a/translate-all.c b/translate-all.c
index 5549a85..6b7b46e 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -475,6 +475,10 @@
 #elif defined(__s390x__)
   /* We have a +- 4GB range on the branches; leave some slop.  */
 # define MAX_CODE_GEN_BUFFER_SIZE  (3ul * 1024 * 1024 * 1024)
+#elif defined(__mips__)
+  /* We have a 256MB branch region, but leave room to make sure the
+     main executable is also within that region.  */
+# define MAX_CODE_GEN_BUFFER_SIZE  (128ul * 1024 * 1024)
 #else
 # define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
 #endif
@@ -509,14 +513,47 @@
     return tb_size;
 }
 
+#ifdef __mips__
+/* In order to use J and JAL within the code_gen_buffer, we require
+   that the buffer not cross a 256MB boundary.  */
+static inline bool cross_256mb(void *addr, size_t size)
+{
+    return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
+}
+
+/* We weren't able to allocate a buffer without crossing that boundary,
+   so make do with the larger portion of the buffer that doesn't cross.
+   Returns the new base of the buffer, and adjusts code_gen_buffer_size.  */
+static inline void *split_cross_256mb(void *buf1, size_t size1)
+{
+    void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
+    size_t size2 = buf1 + size1 - buf2;
+
+    size1 = buf2 - buf1;
+    if (size1 < size2) {
+        size1 = size2;
+        buf1 = buf2;
+    }
+
+    tcg_ctx.code_gen_buffer_size = size1;
+    return buf1;
+}
+#endif
+
 #ifdef USE_STATIC_CODE_GEN_BUFFER
 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
     __attribute__((aligned(CODE_GEN_ALIGN)));
 
 static inline void *alloc_code_gen_buffer(void)
 {
-    map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
-    return static_code_gen_buffer;
+    void *buf = static_code_gen_buffer;
+#ifdef __mips__
+    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+        buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
+    }
+#endif
+    map_exec(buf, tcg_ctx.code_gen_buffer_size);
+    return buf;
 }
 #elif defined(USE_MMAP)
 static inline void *alloc_code_gen_buffer(void)
@@ -545,20 +582,76 @@
     start = 0x40000000ul;
 # elif defined(__s390x__)
     start = 0x90000000ul;
+# elif defined(__mips__)
+    /* ??? We ought to more explicitly manage layout for softmmu too.  */
+#  ifdef CONFIG_USER_ONLY
+    start = 0x68000000ul;
+#  elif _MIPS_SIM == _ABI64
+    start = 0x128000000ul;
+#  else
+    start = 0x08000000ul;
+#  endif
 # endif
 
     buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
                PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
-    return buf == MAP_FAILED ? NULL : buf;
+    if (buf == MAP_FAILED) {
+        return NULL;
+    }
+
+#ifdef __mips__
+    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+        /* Try again, with the original still mapped, to avoid re-aquiring
+           that 256mb crossing.  This time don't specify an address.  */
+        size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
+        void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
+                          flags, -1, 0);
+        if (buf2 != MAP_FAILED) {
+            if (!cross_256mb(buf2, size1)) {
+                /* Success!  Use the new buffer.  */
+                munmap(buf, size1);
+                return buf2;
+            }
+            /* Failure.  Work with what we had.  */
+            munmap(buf2, size1);
+        }
+
+        /* Split the original buffer.  Free the smaller half.  */
+        buf2 = split_cross_256mb(buf, size1);
+        size2 = tcg_ctx.code_gen_buffer_size;
+        munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
+        return buf2;
+    }
+#endif
+
+    return buf;
 }
 #else
 static inline void *alloc_code_gen_buffer(void)
 {
     void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
 
-    if (buf) {
-        map_exec(buf, tcg_ctx.code_gen_buffer_size);
+    if (buf == NULL) {
+        return NULL;
     }
+
+#ifdef __mips__
+    if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
+        void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
+        if (buf2 != NULL && !cross_256mb(buf2, size1)) {
+            /* Success!  Use the new buffer.  */
+            free(buf);
+            buf = buf2;
+        } else {
+            /* Failure.  Work with what we had.  Since this is malloc
+               and not mmap, we can't free the other half.  */
+            free(buf2);
+            buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
+        }
+    }
+#endif
+
+    map_exec(buf, tcg_ctx.code_gen_buffer_size);
     return buf;
 }
 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
diff --git a/ui/console.c b/ui/console.c
index 34d1eaa..2ce55a6 100644
--- a/ui/console.c
+++ b/ui/console.c
@@ -143,8 +143,6 @@
     TextCell *cells;
     int text_x[2], text_y[2], cursor_invalidate;
     int echo;
-    bool cursor_visible_phase;
-    QEMUTimer *cursor_timer;
 
     int update_x0;
     int update_y0;
@@ -177,10 +175,14 @@
 static QemuConsole *active_console;
 static QemuConsole *consoles[MAX_CONSOLES];
 static int nb_consoles = 0;
+static bool cursor_visible_phase;
+static QEMUTimer *cursor_timer;
 
 static void text_console_do_init(CharDriverState *chr, DisplayState *ds);
 static void dpy_refresh(DisplayState *s);
 static DisplayState *get_alloc_displaystate(void);
+static void text_console_update_cursor_timer(void);
+static void text_console_update_cursor(void *opaque);
 
 static void gui_update(void *opaque)
 {
@@ -475,6 +477,9 @@
 
 static void invalidate_xy(QemuConsole *s, int x, int y)
 {
+    if (!qemu_console_is_visible(s)) {
+        return;
+    }
     if (s->update_x0 > x * FONT_WIDTH)
         s->update_x0 = x * FONT_WIDTH;
     if (s->update_y0 > y * FONT_HEIGHT)
@@ -490,25 +495,20 @@
     TextCell *c;
     int y1, y2;
 
-    if (!qemu_console_is_visible(s)) {
-        return;
-    }
-
     if (s->ds->have_text) {
         text_update_xy(s, x, y);
     }
 
-    if (s->ds->have_gfx) {
-        y1 = (s->y_base + y) % s->total_height;
-        y2 = y1 - s->y_displayed;
-        if (y2 < 0)
-            y2 += s->total_height;
-        if (y2 < s->height) {
-            c = &s->cells[y1 * s->width + x];
-            vga_putcharxy(s, x, y2, c->ch,
-                          &(c->t_attrib));
-            invalidate_xy(s, x, y2);
-        }
+    y1 = (s->y_base + y) % s->total_height;
+    y2 = y1 - s->y_displayed;
+    if (y2 < 0) {
+        y2 += s->total_height;
+    }
+    if (y2 < s->height) {
+        c = &s->cells[y1 * s->width + x];
+        vga_putcharxy(s, x, y2, c->ch,
+                      &(c->t_attrib));
+        invalidate_xy(s, x, y2);
     }
 }
 
@@ -518,33 +518,28 @@
     int y, y1;
     int x = s->x;
 
-    if (!qemu_console_is_visible(s)) {
-        return;
-    }
-
     if (s->ds->have_text) {
         s->cursor_invalidate = 1;
     }
 
-    if (s->ds->have_gfx) {
-        if (x >= s->width) {
-            x = s->width - 1;
+    if (x >= s->width) {
+        x = s->width - 1;
+    }
+    y1 = (s->y_base + s->y) % s->total_height;
+    y = y1 - s->y_displayed;
+    if (y < 0) {
+        y += s->total_height;
+    }
+    if (y < s->height) {
+        c = &s->cells[y1 * s->width + x];
+        if (show && cursor_visible_phase) {
+            TextAttributes t_attrib = s->t_attrib_default;
+            t_attrib.invers = !(t_attrib.invers); /* invert fg and bg */
+            vga_putcharxy(s, x, y, c->ch, &t_attrib);
+        } else {
+            vga_putcharxy(s, x, y, c->ch, &(c->t_attrib));
         }
-        y1 = (s->y_base + s->y) % s->total_height;
-        y = y1 - s->y_displayed;
-        if (y < 0)
-            y += s->total_height;
-        if (y < s->height) {
-            c = &s->cells[y1 * s->width + x];
-            if (show && s->cursor_visible_phase) {
-                TextAttributes t_attrib = s->t_attrib_default;
-                t_attrib.invers = !(t_attrib.invers); /* invert fg and bg */
-                vga_putcharxy(s, x, y, c->ch, &t_attrib);
-            } else {
-                vga_putcharxy(s, x, y, c->ch, &(c->t_attrib));
-            }
-            invalidate_xy(s, x, y);
-        }
+        invalidate_xy(s, x, y);
     }
 }
 
@@ -554,10 +549,6 @@
     TextCell *c;
     int x, y, y1;
 
-    if (!qemu_console_is_visible(s)) {
-        return;
-    }
-
     if (s->ds->have_text) {
         s->text_x[0] = 0;
         s->text_y[0] = 0;
@@ -566,25 +557,23 @@
         s->cursor_invalidate = 1;
     }
 
-    if (s->ds->have_gfx) {
-        vga_fill_rect(s, 0, 0, surface_width(surface), surface_height(surface),
-                      color_table_rgb[0][COLOR_BLACK]);
-        y1 = s->y_displayed;
-        for (y = 0; y < s->height; y++) {
-            c = s->cells + y1 * s->width;
-            for (x = 0; x < s->width; x++) {
-                vga_putcharxy(s, x, y, c->ch,
-                              &(c->t_attrib));
-                c++;
-            }
-            if (++y1 == s->total_height) {
-                y1 = 0;
-            }
+    vga_fill_rect(s, 0, 0, surface_width(surface), surface_height(surface),
+                  color_table_rgb[0][COLOR_BLACK]);
+    y1 = s->y_displayed;
+    for (y = 0; y < s->height; y++) {
+        c = s->cells + y1 * s->width;
+        for (x = 0; x < s->width; x++) {
+            vga_putcharxy(s, x, y, c->ch,
+                          &(c->t_attrib));
+            c++;
         }
-        console_show_cursor(s, 1);
-        dpy_gfx_update(s, 0, 0,
-                       surface_width(surface), surface_height(surface));
+        if (++y1 == s->total_height) {
+            y1 = 0;
+        }
     }
+    console_show_cursor(s, 1);
+    dpy_gfx_update(s, 0, 0,
+                   surface_width(surface), surface_height(surface));
 }
 
 static void console_scroll(QemuConsole *s, int ydelta)
@@ -640,7 +629,7 @@
             c->t_attrib = s->t_attrib_default;
             c++;
         }
-        if (qemu_console_is_visible(s) && s->y_displayed == s->y_base) {
+        if (s->y_displayed == s->y_base) {
             if (s->ds->have_text) {
                 s->text_x[0] = 0;
                 s->text_y[0] = 0;
@@ -648,18 +637,16 @@
                 s->text_y[1] = s->height - 1;
             }
 
-            if (s->ds->have_gfx) {
-                vga_bitblt(s, 0, FONT_HEIGHT, 0, 0,
-                           s->width * FONT_WIDTH,
-                           (s->height - 1) * FONT_HEIGHT);
-                vga_fill_rect(s, 0, (s->height - 1) * FONT_HEIGHT,
-                              s->width * FONT_WIDTH, FONT_HEIGHT,
-                              color_table_rgb[0][s->t_attrib_default.bgcol]);
-                s->update_x0 = 0;
-                s->update_y0 = 0;
-                s->update_x1 = s->width * FONT_WIDTH;
-                s->update_y1 = s->height * FONT_HEIGHT;
-            }
+            vga_bitblt(s, 0, FONT_HEIGHT, 0, 0,
+                       s->width * FONT_WIDTH,
+                       (s->height - 1) * FONT_HEIGHT);
+            vga_fill_rect(s, 0, (s->height - 1) * FONT_HEIGHT,
+                          s->width * FONT_WIDTH, FONT_HEIGHT,
+                          color_table_rgb[0][s->t_attrib_default.bgcol]);
+            s->update_x0 = 0;
+            s->update_y0 = 0;
+            s->update_x1 = s->width * FONT_WIDTH;
+            s->update_y1 = s->height * FONT_HEIGHT;
         }
     }
 }
@@ -1004,9 +991,6 @@
     if (s) {
         DisplayState *ds = s->ds;
 
-        if (active_console && active_console->cursor_timer) {
-            timer_del(active_console->cursor_timer);
-        }
         active_console = s;
         if (ds->have_gfx) {
             QLIST_FOREACH(dcl, &ds->listeners, next) {
@@ -1023,10 +1007,7 @@
         if (ds->have_text) {
             dpy_text_resize(s, s->width, s->height);
         }
-        if (s->cursor_timer) {
-            timer_mod(s->cursor_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + CONSOLE_CURSOR_PERIOD / 2);
-        }
+        text_console_update_cursor(NULL);
     }
 }
 
@@ -1075,13 +1056,11 @@
 }
 
 /* called when an ascii key is pressed */
-void kbd_put_keysym(int keysym)
+void kbd_put_keysym_console(QemuConsole *s, int keysym)
 {
-    QemuConsole *s;
     uint8_t buf[16], *q;
     int c;
 
-    s = active_console;
     if (!s || (s->console_type == GRAPHIC_CONSOLE))
         return;
 
@@ -1130,6 +1109,44 @@
     }
 }
 
+static const int qcode_to_keysym[Q_KEY_CODE_MAX] = {
+    [Q_KEY_CODE_UP]     = QEMU_KEY_UP,
+    [Q_KEY_CODE_DOWN]   = QEMU_KEY_DOWN,
+    [Q_KEY_CODE_RIGHT]  = QEMU_KEY_RIGHT,
+    [Q_KEY_CODE_LEFT]   = QEMU_KEY_LEFT,
+    [Q_KEY_CODE_HOME]   = QEMU_KEY_HOME,
+    [Q_KEY_CODE_END]    = QEMU_KEY_END,
+    [Q_KEY_CODE_PGUP]   = QEMU_KEY_PAGEUP,
+    [Q_KEY_CODE_PGDN]   = QEMU_KEY_PAGEDOWN,
+    [Q_KEY_CODE_DELETE] = QEMU_KEY_DELETE,
+};
+
+bool kbd_put_qcode_console(QemuConsole *s, int qcode)
+{
+    int keysym;
+
+    keysym = qcode_to_keysym[qcode];
+    if (keysym == 0) {
+        return false;
+    }
+    kbd_put_keysym_console(s, keysym);
+    return true;
+}
+
+void kbd_put_string_console(QemuConsole *s, const char *str, int len)
+{
+    int i;
+
+    for (i = 0; i < len && str[i]; i++) {
+        kbd_put_keysym_console(s, str[i]);
+    }
+}
+
+void kbd_put_keysym(int keysym)
+{
+    kbd_put_keysym_console(active_console, keysym);
+}
+
 static void text_console_invalidate(void *opaque)
 {
     QemuConsole *s = (QemuConsole *) opaque;
@@ -1167,9 +1184,9 @@
     }
 }
 
-static QemuConsole *new_console(DisplayState *ds, console_type_t console_type)
+static QemuConsole *new_console(DisplayState *ds, console_type_t console_type,
+                                uint32_t head)
 {
-    Error *local_err = NULL;
     Object *obj;
     QemuConsole *s;
     int i;
@@ -1179,13 +1196,14 @@
 
     obj = object_new(TYPE_QEMU_CONSOLE);
     s = QEMU_CONSOLE(obj);
+    s->head = head;
     object_property_add_link(obj, "device", TYPE_DEVICE,
                              (Object **)&s->device,
                              object_property_allow_set_link,
                              OBJ_PROP_LINK_UNREF_ON_RELEASE,
-                             &local_err);
+                             &error_abort);
     object_property_add_uint32_ptr(obj, "head",
-                                   &s->head, &local_err);
+                                   &s->head, &error_abort);
 
     if (!active_console || ((active_console->console_type != GRAPHIC_CONSOLE) &&
         (console_type == GRAPHIC_CONSOLE))) {
@@ -1270,19 +1288,18 @@
     return surface;
 }
 
-static DisplaySurface *qemu_create_dummy_surface(void)
+static DisplaySurface *qemu_create_message_surface(int w, int h,
+                                                   const char *msg)
 {
-    static const char msg[] =
-        "This VM has no graphic display device.";
-    DisplaySurface *surface = qemu_create_displaysurface(640, 480);
+    DisplaySurface *surface = qemu_create_displaysurface(w, h);
     pixman_color_t bg = color_table_rgb[0][COLOR_BLACK];
     pixman_color_t fg = color_table_rgb[0][COLOR_WHITE];
     pixman_image_t *glyph;
     int len, x, y, i;
 
     len = strlen(msg);
-    x = (640/FONT_WIDTH  - len) / 2;
-    y = (480/FONT_HEIGHT - 1)   / 2;
+    x = (w / FONT_WIDTH  - len) / 2;
+    y = (h / FONT_HEIGHT - 1)   / 2;
     for (i = 0; i < len; i++) {
         glyph = qemu_pixman_glyph_from_vgafont(FONT_HEIGHT, vgafont16, msg[i]);
         qemu_pixman_glyph_render(glyph, surface->image, &fg, &bg,
@@ -1304,6 +1321,8 @@
 
 void register_displaychangelistener(DisplayChangeListener *dcl)
 {
+    static const char nodev[] =
+        "This VM has no graphic display device.";
     static DisplaySurface *dummy;
     QemuConsole *con;
 
@@ -1322,11 +1341,12 @@
             dcl->ops->dpy_gfx_switch(dcl, con->surface);
         } else {
             if (!dummy) {
-                dummy = qemu_create_dummy_surface();
+                dummy = qemu_create_message_surface(640, 480, nodev);
             }
             dcl->ops->dpy_gfx_switch(dcl, dummy);
         }
     }
+    text_console_update_cursor(NULL);
 }
 
 void update_displaychangelistener(DisplayChangeListener *dcl,
@@ -1550,6 +1570,8 @@
 {
     if (!display_state) {
         display_state = g_new0(DisplayState, 1);
+        cursor_timer = timer_new_ms(QEMU_CLOCK_REALTIME,
+                                    text_console_update_cursor, NULL);
     }
     return display_state;
 }
@@ -1560,7 +1582,6 @@
  */
 DisplayState *init_displaystate(void)
 {
-    Error *local_err = NULL;
     gchar *name;
     int i;
 
@@ -1579,7 +1600,7 @@
          * doesn't change any more */
         name = g_strdup_printf("console[%d]", i);
         object_property_add_child(container_get(object_get_root(), "/backend"),
-                                  name, OBJECT(consoles[i]), &local_err);
+                                  name, OBJECT(consoles[i]), &error_abort);
         g_free(name);
     }
 
@@ -1590,7 +1611,8 @@
                                   const GraphicHwOps *hw_ops,
                                   void *opaque)
 {
-    Error *local_err = NULL;
+    static const char noinit[] =
+        "Guest has not initialized the display (yet).";
     int width = 640;
     int height = 480;
     QemuConsole *s;
@@ -1598,17 +1620,15 @@
 
     ds = get_alloc_displaystate();
     trace_console_gfx_new();
-    s = new_console(ds, GRAPHIC_CONSOLE);
+    s = new_console(ds, GRAPHIC_CONSOLE, head);
     s->hw_ops = hw_ops;
     s->hw = opaque;
     if (dev) {
-        object_property_set_link(OBJECT(s), OBJECT(dev),
-                                 "device", &local_err);
-        object_property_set_int(OBJECT(s), head,
-                                "head", &local_err);
+        object_property_set_link(OBJECT(s), OBJECT(dev), "device",
+                                 &error_abort);
     }
 
-    s->surface = qemu_create_displaysurface(width, height);
+    s->surface = qemu_create_message_surface(width, height, noinit);
     return s;
 }
 
@@ -1622,7 +1642,6 @@
 
 QemuConsole *qemu_console_lookup_by_device(DeviceState *dev, uint32_t head)
 {
-    Error *local_err = NULL;
     Object *obj;
     uint32_t h;
     int i;
@@ -1632,12 +1651,12 @@
             continue;
         }
         obj = object_property_get_link(OBJECT(consoles[i]),
-                                       "device", &local_err);
+                                       "device", &error_abort);
         if (DEVICE(obj) != dev) {
             continue;
         }
         h = object_property_get_int(OBJECT(consoles[i]),
-                                    "head", &local_err);
+                                    "head", &error_abort);
         if (h != head) {
             continue;
         }
@@ -1712,14 +1731,32 @@
     s->echo = echo;
 }
 
+static void text_console_update_cursor_timer(void)
+{
+    timer_mod(cursor_timer, qemu_clock_get_ms(QEMU_CLOCK_REALTIME)
+              + CONSOLE_CURSOR_PERIOD / 2);
+}
+
 static void text_console_update_cursor(void *opaque)
 {
-    QemuConsole *s = opaque;
+    QemuConsole *s;
+    int i, count = 0;
 
-    s->cursor_visible_phase = !s->cursor_visible_phase;
-    graphic_hw_invalidate(s);
-    timer_mod(s->cursor_timer,
-                   qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + CONSOLE_CURSOR_PERIOD / 2);
+    cursor_visible_phase = !cursor_visible_phase;
+
+    for (i = 0; i < nb_consoles; i++) {
+        s = consoles[i];
+        if (qemu_console_is_graphic(s) ||
+            !qemu_console_is_visible(s)) {
+            continue;
+        }
+        count++;
+        graphic_hw_invalidate(s);
+    }
+
+    if (count) {
+        text_console_update_cursor_timer();
+    }
 }
 
 static const GraphicHwOps text_console_ops = {
@@ -1755,9 +1792,6 @@
         s->surface = qemu_create_displaysurface(g_width, g_height);
     }
 
-    s->cursor_timer =
-        timer_new_ms(QEMU_CLOCK_REALTIME, text_console_update_cursor, s);
-
     s->hw_ops = &text_console_ops;
     s->hw = s;
 
@@ -1811,9 +1845,9 @@
 
     trace_console_txt_new(width, height);
     if (width == 0 || height == 0) {
-        s = new_console(NULL, TEXT_CONSOLE);
+        s = new_console(NULL, TEXT_CONSOLE, 0);
     } else {
-        s = new_console(NULL, TEXT_CONSOLE_FIXED_SIZE);
+        s = new_console(NULL, TEXT_CONSOLE_FIXED_SIZE, 0);
         s->surface = qemu_create_displaysurface(width, height);
     }
 
diff --git a/ui/curses.c b/ui/curses.c
index b044790..8edb038 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -277,31 +277,41 @@
              * events, we need to emit both for each key received */
             if (keycode & SHIFT) {
                 qemu_input_event_send_key_number(NULL, SHIFT_CODE, true);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & CNTRL) {
                 qemu_input_event_send_key_number(NULL, CNTRL_CODE, true);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & ALT) {
                 qemu_input_event_send_key_number(NULL, ALT_CODE, true);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & ALTGR) {
                 qemu_input_event_send_key_number(NULL, GREY | ALT_CODE, true);
+                qemu_input_event_send_key_delay(0);
             }
 
-            qemu_input_event_send_key_number(NULL, keycode, true);
-            qemu_input_event_send_key_number(NULL, keycode, false);
+            qemu_input_event_send_key_number(NULL, keycode & KEY_MASK, true);
+            qemu_input_event_send_key_delay(0);
+            qemu_input_event_send_key_number(NULL, keycode & KEY_MASK, false);
+            qemu_input_event_send_key_delay(0);
 
             if (keycode & ALTGR) {
                 qemu_input_event_send_key_number(NULL, GREY | ALT_CODE, false);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & ALT) {
                 qemu_input_event_send_key_number(NULL, ALT_CODE, false);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & CNTRL) {
                 qemu_input_event_send_key_number(NULL, CNTRL_CODE, false);
+                qemu_input_event_send_key_delay(0);
             }
             if (keycode & SHIFT) {
                 qemu_input_event_send_key_number(NULL, SHIFT_CODE, false);
+                qemu_input_event_send_key_delay(0);
             }
         } else {
             keysym = curses2qemu[chr];
diff --git a/ui/gtk.c b/ui/gtk.c
index 9f5061a..b908936 100644
--- a/ui/gtk.c
+++ b/ui/gtk.c
@@ -67,13 +67,38 @@
 #include "x_keymap.h"
 #include "keymaps.h"
 #include "sysemu/char.h"
+#include "qom/object.h"
+#ifndef _WIN32
+#include <gdk/gdkx.h>
+#include <X11/XKBlib.h>
+#endif
 
 #define MAX_VCS 10
+#define VC_WINDOW_X_MIN  320
+#define VC_WINDOW_Y_MIN  240
+#define VC_TERM_X_MIN     80
+#define VC_TERM_Y_MIN     25
+#define VC_SCALE_MIN    0.25
+#define VC_SCALE_STEP   0.25
 
 #if !defined(CONFIG_VTE)
 # define VTE_CHECK_VERSION(a, b, c) 0
 #endif
 
+#if defined(CONFIG_VTE) && !GTK_CHECK_VERSION(3, 0, 0)
+/*
+ * The gtk2 vte terminal widget seriously messes up the window resize
+ * for some reason.  You basically can't make the qemu window smaller
+ * any more because the toplevel window geoemtry hints are overridden.
+ *
+ * Workaround that by hiding all vte widgets, except the one in the
+ * current tab.
+ *
+ * Luckily everything works smooth in gtk3.
+ */
+# define VTE_RESIZE_HACK 1
+#endif
+
 /* Compatibility define to let us build on both Gtk2 and Gtk3 */
 #if GTK_CHECK_VERSION(3, 0, 0)
 static inline void gdk_drawable_get_size(GdkWindow *w, gint *ww, gint *wh)
@@ -105,18 +130,48 @@
     0x2a, 0x36, 0x1d, 0x9d, 0x38, 0xb8, 0xdb, 0xdd,
 };
 
-typedef struct VirtualConsole
-{
-    GtkWidget *menu_item;
-    GtkWidget *terminal;
+typedef struct GtkDisplayState GtkDisplayState;
+
+typedef struct VirtualGfxConsole {
+    GtkWidget *drawing_area;
+    DisplayChangeListener dcl;
+    DisplaySurface *ds;
+    pixman_image_t *convert;
+    cairo_surface_t *surface;
+    double scale_x;
+    double scale_y;
+} VirtualGfxConsole;
+
 #if defined(CONFIG_VTE)
-    GtkWidget *scrolled_window;
+typedef struct VirtualVteConsole {
+    GtkWidget *box;
+    GtkWidget *scrollbar;
+    GtkWidget *terminal;
     CharDriverState *chr;
+} VirtualVteConsole;
 #endif
+
+typedef enum VirtualConsoleType {
+    GD_VC_GFX,
+    GD_VC_VTE,
+} VirtualConsoleType;
+
+typedef struct VirtualConsole {
+    GtkDisplayState *s;
+    char *label;
+    GtkWidget *window;
+    GtkWidget *menu_item;
+    GtkWidget *tab_item;
+    VirtualConsoleType type;
+    union {
+        VirtualGfxConsole gfx;
+#if defined(CONFIG_VTE)
+        VirtualVteConsole vte;
+#endif
+    };
 } VirtualConsole;
 
-typedef struct GtkDisplayState
-{
+struct GtkDisplayState {
     GtkWidget *window;
 
     GtkWidget *menu_bar;
@@ -139,29 +194,24 @@
     GtkWidget *zoom_fit_item;
     GtkWidget *grab_item;
     GtkWidget *grab_on_hover_item;
-    GtkWidget *vga_item;
 
     int nb_vcs;
     VirtualConsole vc[MAX_VCS];
 
     GtkWidget *show_tabs_item;
+    GtkWidget *untabify_item;
 
     GtkWidget *vbox;
     GtkWidget *notebook;
-    GtkWidget *drawing_area;
-    cairo_surface_t *surface;
-    pixman_image_t *convert;
-    DisplayChangeListener dcl;
-    DisplaySurface *ds;
     int button_mask;
     gboolean last_set;
     int last_x;
     int last_y;
     int grab_x_root;
     int grab_y_root;
+    VirtualConsole *kbd_owner;
+    VirtualConsole *ptr_owner;
 
-    double scale_x;
-    double scale_y;
     gboolean full_screen;
 
     GdkCursor *null_cursor;
@@ -171,12 +221,52 @@
     bool external_pause_update;
 
     bool modifier_pressed[ARRAY_SIZE(modifier_keycode)];
-} GtkDisplayState;
+    bool has_evdev;
+};
 
-static GtkDisplayState *global_state;
+static void gd_grab_pointer(VirtualConsole *vc);
+static void gd_ungrab_pointer(GtkDisplayState *s);
 
 /** Utility Functions **/
 
+static VirtualConsole *gd_vc_find_by_menu(GtkDisplayState *s)
+{
+    VirtualConsole *vc;
+    gint i;
+
+    for (i = 0; i < s->nb_vcs; i++) {
+        vc = &s->vc[i];
+        if (gtk_check_menu_item_get_active
+            (GTK_CHECK_MENU_ITEM(vc->menu_item))) {
+            return vc;
+        }
+    }
+    return NULL;
+}
+
+static VirtualConsole *gd_vc_find_by_page(GtkDisplayState *s, gint page)
+{
+    VirtualConsole *vc;
+    gint i, p;
+
+    for (i = 0; i < s->nb_vcs; i++) {
+        vc = &s->vc[i];
+        p = gtk_notebook_page_num(GTK_NOTEBOOK(s->notebook), vc->tab_item);
+        if (p == page) {
+            return vc;
+        }
+    }
+    return NULL;
+}
+
+static VirtualConsole *gd_vc_find_current(GtkDisplayState *s)
+{
+    gint page;
+
+    page = gtk_notebook_get_current_page(GTK_NOTEBOOK(s->notebook));
+    return gd_vc_find_by_page(s, page);
+}
+
 static bool gd_is_grab_active(GtkDisplayState *s)
 {
     return gtk_check_menu_item_get_active(GTK_CHECK_MENU_ITEM(s->grab_item));
@@ -187,22 +277,17 @@
     return gtk_check_menu_item_get_active(GTK_CHECK_MENU_ITEM(s->grab_on_hover_item));
 }
 
-static bool gd_on_vga(GtkDisplayState *s)
+static void gd_update_cursor(VirtualConsole *vc)
 {
-    return gtk_notebook_get_current_page(GTK_NOTEBOOK(s->notebook)) == 0;
-}
-
-static void gd_update_cursor(GtkDisplayState *s, gboolean override)
-{
+    GtkDisplayState *s = vc->s;
     GdkWindow *window;
-    bool on_vga;
 
-    window = gtk_widget_get_window(GTK_WIDGET(s->drawing_area));
+    if (vc->type != GD_VC_GFX) {
+        return;
+    }
 
-    on_vga = gd_on_vga(s);
-
-    if ((override || on_vga) &&
-        (s->full_screen || qemu_input_is_absolute() || gd_is_grab_active(s))) {
+    window = gtk_widget_get_window(GTK_WIDGET(vc->gfx.drawing_area));
+    if (s->full_screen || qemu_input_is_absolute() || s->ptr_owner == vc) {
         gdk_window_set_cursor(window, s->null_cursor);
     } else {
         gdk_window_set_cursor(window, NULL);
@@ -212,11 +297,20 @@
 static void gd_update_caption(GtkDisplayState *s)
 {
     const char *status = "";
+    gchar *prefix;
     gchar *title;
     const char *grab = "";
     bool is_paused = !runstate_is_running();
+    int i;
 
-    if (gd_is_grab_active(s)) {
+    if (qemu_name) {
+        prefix = g_strdup_printf("QEMU (%s)", qemu_name);
+    } else {
+        prefix = g_strdup_printf("QEMU");
+    }
+
+    if (s->ptr_owner != NULL &&
+        s->ptr_owner->window == NULL) {
         grab = _(" - Press Ctrl+Alt+G to release grab");
     }
 
@@ -228,60 +322,100 @@
                                    is_paused);
     s->external_pause_update = false;
 
-    if (qemu_name) {
-        title = g_strdup_printf("QEMU (%s)%s%s", qemu_name, status, grab);
-    } else {
-        title = g_strdup_printf("QEMU%s%s", status, grab);
-    }
-
+    title = g_strdup_printf("%s%s%s", prefix, status, grab);
     gtk_window_set_title(GTK_WINDOW(s->window), title);
-
     g_free(title);
+
+    for (i = 0; i < s->nb_vcs; i++) {
+        VirtualConsole *vc = &s->vc[i];
+
+        if (!vc->window) {
+            continue;
+        }
+        title = g_strdup_printf("%s: %s%s%s", prefix, vc->label,
+                                vc == s->kbd_owner ? " +kbd" : "",
+                                vc == s->ptr_owner ? " +ptr" : "");
+        gtk_window_set_title(GTK_WINDOW(vc->window), title);
+        g_free(title);
+    }
+
+    g_free(prefix);
 }
 
-static void gd_update_windowsize(GtkDisplayState *s)
+static void gd_update_geometry_hints(VirtualConsole *vc)
 {
-    if (!s->full_screen) {
-        GtkRequisition req;
-        double sx, sy;
+    GtkDisplayState *s = vc->s;
+    GdkWindowHints mask = 0;
+    GdkGeometry geo = {};
+    GtkWidget *geo_widget = NULL;
+    GtkWindow *geo_window;
 
+    if (vc->type == GD_VC_GFX) {
         if (s->free_scale) {
-            sx = s->scale_x;
-            sy = s->scale_y;
-
-            s->scale_y = 1.0;
-            s->scale_x = 1.0;
+            geo.min_width  = surface_width(vc->gfx.ds) * VC_SCALE_MIN;
+            geo.min_height = surface_height(vc->gfx.ds) * VC_SCALE_MIN;
+            mask |= GDK_HINT_MIN_SIZE;
         } else {
-            sx = 1.0;
-            sy = 1.0;
+            geo.min_width  = surface_width(vc->gfx.ds) * vc->gfx.scale_x;
+            geo.min_height = surface_height(vc->gfx.ds) * vc->gfx.scale_y;
+            mask |= GDK_HINT_MIN_SIZE;
         }
+        geo_widget = vc->gfx.drawing_area;
+        gtk_widget_set_size_request(geo_widget, geo.min_width, geo.min_height);
 
-        gtk_widget_set_size_request(s->drawing_area,
-                                    surface_width(s->ds) * s->scale_x,
-                                    surface_height(s->ds) * s->scale_y);
-#if GTK_CHECK_VERSION(3, 0, 0)
-        gtk_widget_get_preferred_size(s->vbox, NULL, &req);
-#else
-        gtk_widget_size_request(s->vbox, &req);
+#if defined(CONFIG_VTE)
+    } else if (vc->type == GD_VC_VTE) {
+        VteTerminal *term = VTE_TERMINAL(vc->vte.terminal);
+        GtkBorder *ib;
+
+        geo.width_inc  = vte_terminal_get_char_width(term);
+        geo.height_inc = vte_terminal_get_char_height(term);
+        mask |= GDK_HINT_RESIZE_INC;
+        geo.base_width  = geo.width_inc;
+        geo.base_height = geo.height_inc;
+        mask |= GDK_HINT_BASE_SIZE;
+        geo.min_width  = geo.width_inc * VC_TERM_X_MIN;
+        geo.min_height = geo.height_inc * VC_TERM_Y_MIN;
+        mask |= GDK_HINT_MIN_SIZE;
+        gtk_widget_style_get(vc->vte.terminal, "inner-border", &ib, NULL);
+        geo.base_width  += ib->left + ib->right;
+        geo.base_height += ib->top + ib->bottom;
+        geo.min_width   += ib->left + ib->right;
+        geo.min_height  += ib->top + ib->bottom;
+        geo_widget = vc->vte.terminal;
 #endif
+    }
 
-        gtk_window_resize(GTK_WINDOW(s->window),
-                          req.width * sx, req.height * sy);
+    geo_window = GTK_WINDOW(vc->window ? vc->window : s->window);
+    gtk_window_set_geometry_hints(geo_window, geo_widget, &geo, mask);
+}
+
+static void gd_update_windowsize(VirtualConsole *vc)
+{
+    GtkDisplayState *s = vc->s;
+
+    gd_update_geometry_hints(vc);
+
+    if (vc->type == GD_VC_GFX && !s->full_screen && !s->free_scale) {
+        gtk_window_resize(GTK_WINDOW(vc->window ? vc->window : s->window),
+                          VC_WINDOW_X_MIN, VC_WINDOW_Y_MIN);
     }
 }
 
-static void gd_update_full_redraw(GtkDisplayState *s)
+static void gd_update_full_redraw(VirtualConsole *vc)
 {
+    GtkWidget *area = vc->gfx.drawing_area;
     int ww, wh;
-    gdk_drawable_get_size(gtk_widget_get_window(s->drawing_area), &ww, &wh);
-    gtk_widget_queue_draw_area(s->drawing_area, 0, 0, ww, wh);
+    gdk_drawable_get_size(gtk_widget_get_window(area), &ww, &wh);
+    gtk_widget_queue_draw_area(area, 0, 0, ww, wh);
 }
 
 static void gtk_release_modifiers(GtkDisplayState *s)
 {
+    VirtualConsole *vc = gd_vc_find_current(s);
     int i, keycode;
 
-    if (!gd_on_vga(s)) {
+    if (vc->type != GD_VC_GFX) {
         return;
     }
     for (i = 0; i < ARRAY_SIZE(modifier_keycode); i++) {
@@ -289,7 +423,7 @@
         if (!s->modifier_pressed[i]) {
             continue;
         }
-        qemu_input_event_send_key_number(s->dcl.con, keycode, false);
+        qemu_input_event_send_key_number(vc->gfx.dcl.con, keycode, false);
         s->modifier_pressed[i] = false;
     }
 }
@@ -299,29 +433,31 @@
 static void gd_update(DisplayChangeListener *dcl,
                       int x, int y, int w, int h)
 {
-    GtkDisplayState *s = container_of(dcl, GtkDisplayState, dcl);
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     int x1, x2, y1, y2;
     int mx, my;
     int fbw, fbh;
     int ww, wh;
 
-    trace_gd_update(x, y, w, h);
+    trace_gd_update(vc->label, x, y, w, h);
 
-    if (s->convert) {
-        pixman_image_composite(PIXMAN_OP_SRC, s->ds->image, NULL, s->convert,
+    if (vc->gfx.convert) {
+        pixman_image_composite(PIXMAN_OP_SRC, vc->gfx.ds->image,
+                               NULL, vc->gfx.convert,
                                x, y, 0, 0, x, y, w, h);
     }
 
-    x1 = floor(x * s->scale_x);
-    y1 = floor(y * s->scale_y);
+    x1 = floor(x * vc->gfx.scale_x);
+    y1 = floor(y * vc->gfx.scale_y);
 
-    x2 = ceil(x * s->scale_x + w * s->scale_x);
-    y2 = ceil(y * s->scale_y + h * s->scale_y);
+    x2 = ceil(x * vc->gfx.scale_x + w * vc->gfx.scale_x);
+    y2 = ceil(y * vc->gfx.scale_y + h * vc->gfx.scale_y);
 
-    fbw = surface_width(s->ds) * s->scale_x;
-    fbh = surface_height(s->ds) * s->scale_y;
+    fbw = surface_width(vc->gfx.ds) * vc->gfx.scale_x;
+    fbh = surface_height(vc->gfx.ds) * vc->gfx.scale_y;
 
-    gdk_drawable_get_size(gtk_widget_get_window(s->drawing_area), &ww, &wh);
+    gdk_drawable_get_size(gtk_widget_get_window(vc->gfx.drawing_area),
+                          &ww, &wh);
 
     mx = my = 0;
     if (ww > fbw) {
@@ -331,7 +467,8 @@
         my = (wh - fbh) / 2;
     }
 
-    gtk_widget_queue_draw_area(s->drawing_area, mx + x1, my + y1, (x2 - x1), (y2 - y1));
+    gtk_widget_queue_draw_area(vc->gfx.drawing_area,
+                               mx + x1, my + y1, (x2 - x1), (y2 - y1));
 }
 
 static void gd_refresh(DisplayChangeListener *dcl)
@@ -343,7 +480,7 @@
 static void gd_mouse_set(DisplayChangeListener *dcl,
                          int x, int y, int visible)
 {
-    GtkDisplayState *s = container_of(dcl, GtkDisplayState, dcl);
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     GdkDisplay *dpy;
     GdkDeviceManager *mgr;
     gint x_root, y_root;
@@ -352,29 +489,29 @@
         return;
     }
 
-    dpy = gtk_widget_get_display(s->drawing_area);
+    dpy = gtk_widget_get_display(vc->gfx.drawing_area);
     mgr = gdk_display_get_device_manager(dpy);
-    gdk_window_get_root_coords(gtk_widget_get_window(s->drawing_area),
+    gdk_window_get_root_coords(gtk_widget_get_window(vc->gfx.drawing_area),
                                x, y, &x_root, &y_root);
     gdk_device_warp(gdk_device_manager_get_client_pointer(mgr),
-                    gtk_widget_get_screen(s->drawing_area),
+                    gtk_widget_get_screen(vc->gfx.drawing_area),
                     x_root, y_root);
 }
 #else
 static void gd_mouse_set(DisplayChangeListener *dcl,
                          int x, int y, int visible)
 {
-    GtkDisplayState *s = container_of(dcl, GtkDisplayState, dcl);
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     gint x_root, y_root;
 
     if (qemu_input_is_absolute()) {
         return;
     }
 
-    gdk_window_get_root_coords(gtk_widget_get_window(s->drawing_area),
+    gdk_window_get_root_coords(gtk_widget_get_window(vc->gfx.drawing_area),
                                x, y, &x_root, &y_root);
-    gdk_display_warp_pointer(gtk_widget_get_display(s->drawing_area),
-                             gtk_widget_get_screen(s->drawing_area),
+    gdk_display_warp_pointer(gtk_widget_get_display(vc->gfx.drawing_area),
+                             gtk_widget_get_screen(vc->gfx.drawing_area),
                              x_root, y_root);
 }
 #endif
@@ -382,7 +519,7 @@
 static void gd_cursor_define(DisplayChangeListener *dcl,
                              QEMUCursor *c)
 {
-    GtkDisplayState *s = container_of(dcl, GtkDisplayState, dcl);
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     GdkPixbuf *pixbuf;
     GdkCursor *cursor;
 
@@ -390,9 +527,10 @@
                                       GDK_COLORSPACE_RGB, true, 8,
                                       c->width, c->height, c->width * 4,
                                       NULL, NULL);
-    cursor = gdk_cursor_new_from_pixbuf(gtk_widget_get_display(s->drawing_area),
-                                        pixbuf, c->hot_x, c->hot_y);
-    gdk_window_set_cursor(gtk_widget_get_window(s->drawing_area), cursor);
+    cursor = gdk_cursor_new_from_pixbuf
+        (gtk_widget_get_display(vc->gfx.drawing_area),
+         pixbuf, c->hot_x, c->hot_y);
+    gdk_window_set_cursor(gtk_widget_get_window(vc->gfx.drawing_area), cursor);
     g_object_unref(pixbuf);
 #if !GTK_CHECK_VERSION(3, 0, 0)
     gdk_cursor_unref(cursor);
@@ -404,25 +542,25 @@
 static void gd_switch(DisplayChangeListener *dcl,
                       DisplaySurface *surface)
 {
-    GtkDisplayState *s = container_of(dcl, GtkDisplayState, dcl);
+    VirtualConsole *vc = container_of(dcl, VirtualConsole, gfx.dcl);
     bool resized = true;
 
-    trace_gd_switch(surface_width(surface), surface_height(surface));
+    trace_gd_switch(vc->label, surface_width(surface), surface_height(surface));
 
-    if (s->surface) {
-        cairo_surface_destroy(s->surface);
+    if (vc->gfx.surface) {
+        cairo_surface_destroy(vc->gfx.surface);
     }
 
-    if (s->ds &&
-        surface_width(s->ds) == surface_width(surface) &&
-        surface_height(s->ds) == surface_height(surface)) {
+    if (vc->gfx.ds &&
+        surface_width(vc->gfx.ds) == surface_width(surface) &&
+        surface_height(vc->gfx.ds) == surface_height(surface)) {
         resized = false;
     }
-    s->ds = surface;
+    vc->gfx.ds = surface;
 
-    if (s->convert) {
-        pixman_image_unref(s->convert);
-        s->convert = NULL;
+    if (vc->gfx.convert) {
+        pixman_image_unref(vc->gfx.convert);
+        vc->gfx.convert = NULL;
     }
 
     if (surface->format == PIXMAN_x8r8g8b8) {
@@ -432,7 +570,7 @@
          * No need to convert, use surface directly.  Should be the
          * common case as this is qemu_default_pixelformat(32) too.
          */
-        s->surface = cairo_image_surface_create_for_data
+        vc->gfx.surface = cairo_image_surface_create_for_data
             (surface_data(surface),
              CAIRO_FORMAT_RGB24,
              surface_width(surface),
@@ -440,26 +578,27 @@
              surface_stride(surface));
     } else {
         /* Must convert surface, use pixman to do it. */
-        s->convert = pixman_image_create_bits(PIXMAN_x8r8g8b8,
-                                              surface_width(surface),
-                                              surface_height(surface),
-                                              NULL, 0);
-        s->surface = cairo_image_surface_create_for_data
-            ((void *)pixman_image_get_data(s->convert),
+        vc->gfx.convert = pixman_image_create_bits(PIXMAN_x8r8g8b8,
+                                                   surface_width(surface),
+                                                   surface_height(surface),
+                                                   NULL, 0);
+        vc->gfx.surface = cairo_image_surface_create_for_data
+            ((void *)pixman_image_get_data(vc->gfx.convert),
              CAIRO_FORMAT_RGB24,
-             pixman_image_get_width(s->convert),
-             pixman_image_get_height(s->convert),
-             pixman_image_get_stride(s->convert));
-        pixman_image_composite(PIXMAN_OP_SRC, s->ds->image, NULL, s->convert,
+             pixman_image_get_width(vc->gfx.convert),
+             pixman_image_get_height(vc->gfx.convert),
+             pixman_image_get_stride(vc->gfx.convert));
+        pixman_image_composite(PIXMAN_OP_SRC, vc->gfx.ds->image,
+                               NULL, vc->gfx.convert,
                                0, 0, 0, 0, 0, 0,
-                               pixman_image_get_width(s->convert),
-                               pixman_image_get_height(s->convert));
+                               pixman_image_get_width(vc->gfx.convert),
+                               pixman_image_get_height(vc->gfx.convert));
     }
 
     if (resized) {
-        gd_update_windowsize(s);
+        gd_update_windowsize(vc);
     } else {
-        gd_update_full_redraw(s);
+        gd_update_full_redraw(vc);
     }
 }
 
@@ -475,6 +614,7 @@
 static void gd_mouse_mode_change(Notifier *notify, void *data)
 {
     GtkDisplayState *s;
+    int i;
 
     s = container_of(notify, GtkDisplayState, mouse_mode_notifier);
     /* release the grab at switching to absolute mode */
@@ -482,7 +622,10 @@
         gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                        FALSE);
     }
-    gd_update_cursor(s, FALSE);
+    for (i = 0; i < s->nb_vcs; i++) {
+        VirtualConsole *vc = &s->vc[i];
+        gd_update_cursor(vc);
+    }
 }
 
 /** GTK Events **/
@@ -491,9 +634,15 @@
                                 void *opaque)
 {
     GtkDisplayState *s = opaque;
+    int i;
 
     if (!no_quit) {
-        unregister_displaychangelistener(&s->dcl);
+        for (i = 0; i < s->nb_vcs; i++) {
+            if (s->vc[i].type != GD_VC_GFX) {
+                continue;
+            }
+            unregister_displaychangelistener(&s->vc[i].gfx.dcl);
+        }
         qmp_quit(NULL);
         return FALSE;
     }
@@ -503,7 +652,8 @@
 
 static gboolean gd_draw_event(GtkWidget *widget, cairo_t *cr, void *opaque)
 {
-    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
     int mx, my;
     int ww, wh;
     int fbw, fbh;
@@ -512,25 +662,25 @@
         return FALSE;
     }
 
-    fbw = surface_width(s->ds);
-    fbh = surface_height(s->ds);
+    fbw = surface_width(vc->gfx.ds);
+    fbh = surface_height(vc->gfx.ds);
 
     gdk_drawable_get_size(gtk_widget_get_window(widget), &ww, &wh);
 
     if (s->full_screen) {
-        s->scale_x = (double)ww / fbw;
-        s->scale_y = (double)wh / fbh;
+        vc->gfx.scale_x = (double)ww / fbw;
+        vc->gfx.scale_y = (double)wh / fbh;
     } else if (s->free_scale) {
         double sx, sy;
 
         sx = (double)ww / fbw;
         sy = (double)wh / fbh;
 
-        s->scale_x = s->scale_y = MIN(sx, sy);
+        vc->gfx.scale_x = vc->gfx.scale_y = MIN(sx, sy);
     }
 
-    fbw *= s->scale_x;
-    fbh *= s->scale_y;
+    fbw *= vc->gfx.scale_x;
+    fbh *= vc->gfx.scale_y;
 
     mx = my = 0;
     if (ww > fbw) {
@@ -551,8 +701,9 @@
                     -1 * fbw, fbh);
     cairo_fill(cr);
 
-    cairo_scale(cr, s->scale_x, s->scale_y);
-    cairo_set_source_surface(cr, s->surface, mx / s->scale_x, my / s->scale_y);
+    cairo_scale(cr, vc->gfx.scale_x, vc->gfx.scale_y);
+    cairo_set_source_surface(cr, vc->gfx.surface,
+                             mx / vc->gfx.scale_x, my / vc->gfx.scale_y);
     cairo_paint(cr);
 
     return TRUE;
@@ -584,16 +735,18 @@
 static gboolean gd_motion_event(GtkWidget *widget, GdkEventMotion *motion,
                                 void *opaque)
 {
-    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
     int x, y;
     int mx, my;
     int fbh, fbw;
     int ww, wh;
 
-    fbw = surface_width(s->ds) * s->scale_x;
-    fbh = surface_height(s->ds) * s->scale_y;
+    fbw = surface_width(vc->gfx.ds) * vc->gfx.scale_x;
+    fbh = surface_height(vc->gfx.ds) * vc->gfx.scale_y;
 
-    gdk_drawable_get_size(gtk_widget_get_window(s->drawing_area), &ww, &wh);
+    gdk_drawable_get_size(gtk_widget_get_window(vc->gfx.drawing_area),
+                          &ww, &wh);
 
     mx = my = 0;
     if (ww > fbw) {
@@ -603,31 +756,31 @@
         my = (wh - fbh) / 2;
     }
 
-    x = (motion->x - mx) / s->scale_x;
-    y = (motion->y - my) / s->scale_y;
+    x = (motion->x - mx) / vc->gfx.scale_x;
+    y = (motion->y - my) / vc->gfx.scale_y;
 
     if (qemu_input_is_absolute()) {
         if (x < 0 || y < 0 ||
-            x >= surface_width(s->ds) ||
-            y >= surface_height(s->ds)) {
+            x >= surface_width(vc->gfx.ds) ||
+            y >= surface_height(vc->gfx.ds)) {
             return TRUE;
         }
-        qemu_input_queue_abs(s->dcl.con, INPUT_AXIS_X, x,
-                             surface_width(s->ds));
-        qemu_input_queue_abs(s->dcl.con, INPUT_AXIS_Y, y,
-                             surface_height(s->ds));
+        qemu_input_queue_abs(vc->gfx.dcl.con, INPUT_AXIS_X, x,
+                             surface_width(vc->gfx.ds));
+        qemu_input_queue_abs(vc->gfx.dcl.con, INPUT_AXIS_Y, y,
+                             surface_height(vc->gfx.ds));
         qemu_input_event_sync();
-    } else if (s->last_set && gd_is_grab_active(s)) {
-        qemu_input_queue_rel(s->dcl.con, INPUT_AXIS_X, x - s->last_x);
-        qemu_input_queue_rel(s->dcl.con, INPUT_AXIS_Y, y - s->last_y);
+    } else if (s->last_set && s->ptr_owner == vc) {
+        qemu_input_queue_rel(vc->gfx.dcl.con, INPUT_AXIS_X, x - s->last_x);
+        qemu_input_queue_rel(vc->gfx.dcl.con, INPUT_AXIS_Y, y - s->last_y);
         qemu_input_event_sync();
     }
     s->last_x = x;
     s->last_y = y;
     s->last_set = TRUE;
 
-    if (!qemu_input_is_absolute() && gd_is_grab_active(s)) {
-        GdkScreen *screen = gtk_widget_get_screen(s->drawing_area);
+    if (!qemu_input_is_absolute() && s->ptr_owner == vc) {
+        GdkScreen *screen = gtk_widget_get_screen(vc->gfx.drawing_area);
         int x = (int)motion->x_root;
         int y = (int)motion->y_root;
 
@@ -669,14 +822,21 @@
 static gboolean gd_button_event(GtkWidget *widget, GdkEventButton *button,
                                 void *opaque)
 {
-    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
     InputButton btn;
 
     /* implicitly grab the input at the first click in the relative mode */
     if (button->button == 1 && button->type == GDK_BUTTON_PRESS &&
-        !qemu_input_is_absolute() && !gd_is_grab_active(s)) {
-        gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
-                                       TRUE);
+        !qemu_input_is_absolute() && s->ptr_owner != vc) {
+        gd_ungrab_pointer(s);
+        if (!vc->window) {
+            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
+                                           TRUE);
+        } else {
+            gd_grab_pointer(vc);
+            gd_update_caption(s);
+        }
         return TRUE;
     }
 
@@ -690,7 +850,8 @@
         return TRUE;
     }
 
-    qemu_input_queue_btn(s->dcl.con, btn, button->type == GDK_BUTTON_PRESS);
+    qemu_input_queue_btn(vc->gfx.dcl.con, btn,
+                         button->type == GDK_BUTTON_PRESS);
     qemu_input_event_sync();
     return TRUE;
 }
@@ -698,7 +859,7 @@
 static gboolean gd_scroll_event(GtkWidget *widget, GdkEventScroll *scroll,
                                 void *opaque)
 {
-    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = opaque;
     InputButton btn;
 
     if (scroll->direction == GDK_SCROLL_UP) {
@@ -709,16 +870,17 @@
         return TRUE;
     }
 
-    qemu_input_queue_btn(s->dcl.con, btn, true);
+    qemu_input_queue_btn(vc->gfx.dcl.con, btn, true);
     qemu_input_event_sync();
-    qemu_input_queue_btn(s->dcl.con, btn, false);
+    qemu_input_queue_btn(vc->gfx.dcl.con, btn, false);
     qemu_input_event_sync();
     return TRUE;
 }
 
 static gboolean gd_key_event(GtkWidget *widget, GdkEventKey *key, void *opaque)
 {
-    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
     int gdk_keycode = key->hardware_keycode;
     int i;
 
@@ -737,7 +899,11 @@
     } else if (gdk_keycode < 97) {
         qemu_keycode = gdk_keycode - 8;
     } else if (gdk_keycode < 158) {
-        qemu_keycode = translate_evdev_keycode(gdk_keycode - 97);
+        if (s->has_evdev) {
+            qemu_keycode = translate_evdev_keycode(gdk_keycode - 97);
+        } else {
+            qemu_keycode = translate_xfree86_keycode(gdk_keycode - 97);
+        }
     } else if (gdk_keycode == 208) { /* Hiragana_Katakana */
         qemu_keycode = 0x70;
     } else if (gdk_keycode == 211) { /* backslash */
@@ -747,7 +913,7 @@
     }
 #endif
 
-    trace_gd_key_event(gdk_keycode, qemu_keycode,
+    trace_gd_key_event(vc->label, gdk_keycode, qemu_keycode,
                        (key->type == GDK_KEY_PRESS) ? "down" : "up");
 
     for (i = 0; i < ARRAY_SIZE(modifier_keycode); i++) {
@@ -756,7 +922,7 @@
         }
     }
 
-    qemu_input_event_send_key_number(s->dcl.con, qemu_keycode,
+    qemu_input_event_send_key_number(vc->gfx.dcl.con, qemu_keycode,
                                      key->type == GDK_KEY_PRESS);
 
     return TRUE;
@@ -804,19 +970,14 @@
 static void gd_menu_switch_vc(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_by_menu(s);
+    gint page;
 
-    if (gtk_check_menu_item_get_active(GTK_CHECK_MENU_ITEM(s->vga_item))) {
-        gtk_notebook_set_current_page(GTK_NOTEBOOK(s->notebook), 0);
-    } else {
-        int i;
-
-        gtk_release_modifiers(s);
-        for (i = 0; i < s->nb_vcs; i++) {
-            if (gtk_check_menu_item_get_active(GTK_CHECK_MENU_ITEM(s->vc[i].menu_item))) {
-                gtk_notebook_set_current_page(GTK_NOTEBOOK(s->notebook), i + 1);
-                break;
-            }
-        }
+    gtk_release_modifiers(s);
+    if (vc) {
+        page = gtk_notebook_page_num(GTK_NOTEBOOK(s->notebook),
+                                     vc->tab_item);
+        gtk_notebook_set_current_page(GTK_NOTEBOOK(s->notebook), page);
     }
 }
 
@@ -831,94 +992,159 @@
     }
 }
 
+static gboolean gd_tab_window_close(GtkWidget *widget, GdkEvent *event,
+                                    void *opaque)
+{
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
+
+    gtk_widget_set_sensitive(vc->menu_item, true);
+    gtk_widget_reparent(vc->tab_item, s->notebook);
+    gtk_notebook_set_tab_label_text(GTK_NOTEBOOK(s->notebook),
+                                    vc->tab_item, vc->label);
+    gtk_widget_destroy(vc->window);
+    vc->window = NULL;
+    return TRUE;
+}
+
+static gboolean gd_win_grab(void *opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    fprintf(stderr, "%s: %s\n", __func__, vc->label);
+    if (vc->s->ptr_owner) {
+        gd_ungrab_pointer(vc->s);
+    } else {
+        gd_grab_pointer(vc);
+    }
+    gd_update_caption(vc->s);
+    return TRUE;
+}
+
+static void gd_menu_untabify(GtkMenuItem *item, void *opaque)
+{
+    GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
+
+    if (vc->type == GD_VC_GFX) {
+        gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
+                                       FALSE);
+    }
+    if (!vc->window) {
+        gtk_widget_set_sensitive(vc->menu_item, false);
+        vc->window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
+        gtk_widget_reparent(vc->tab_item, vc->window);
+
+        g_signal_connect(vc->window, "delete-event",
+                         G_CALLBACK(gd_tab_window_close), vc);
+        gtk_widget_show_all(vc->window);
+
+        GtkAccelGroup *ag = gtk_accel_group_new();
+        gtk_window_add_accel_group(GTK_WINDOW(vc->window), ag);
+
+        GClosure *cb = g_cclosure_new_swap(G_CALLBACK(gd_win_grab), vc, NULL);
+        gtk_accel_group_connect(ag, GDK_KEY_g, HOTKEY_MODIFIERS, 0, cb);
+
+        gd_update_geometry_hints(vc);
+        gd_update_caption(s);
+    }
+}
+
 static void gd_menu_full_screen(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
     if (!s->full_screen) {
         gtk_notebook_set_show_tabs(GTK_NOTEBOOK(s->notebook), FALSE);
         gtk_widget_set_size_request(s->menu_bar, 0, 0);
-        gtk_widget_set_size_request(s->drawing_area, -1, -1);
-        gtk_window_fullscreen(GTK_WINDOW(s->window));
-        if (gd_on_vga(s)) {
-            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item), TRUE);
+        if (vc->type == GD_VC_GFX) {
+            gtk_widget_set_size_request(vc->gfx.drawing_area, -1, -1);
+            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
+                                           TRUE);
         }
+        gtk_window_fullscreen(GTK_WINDOW(s->window));
         s->full_screen = TRUE;
     } else {
         gtk_window_unfullscreen(GTK_WINDOW(s->window));
         gd_menu_show_tabs(GTK_MENU_ITEM(s->show_tabs_item), s);
         gtk_widget_set_size_request(s->menu_bar, -1, -1);
-        gtk_widget_set_size_request(s->drawing_area,
-                                    surface_width(s->ds),
-                                    surface_height(s->ds));
-        gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item), FALSE);
         s->full_screen = FALSE;
-        s->scale_x = 1.0;
-        s->scale_y = 1.0;
+        if (vc->type == GD_VC_GFX) {
+            vc->gfx.scale_x = 1.0;
+            vc->gfx.scale_y = 1.0;
+            gd_update_windowsize(vc);
+            gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
+                                           FALSE);
+        }
     }
 
-    gd_update_cursor(s, FALSE);
+    gd_update_cursor(vc);
 }
 
 static void gd_menu_zoom_in(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
     gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->zoom_fit_item),
                                    FALSE);
 
-    s->scale_x += .25;
-    s->scale_y += .25;
+    vc->gfx.scale_x += VC_SCALE_STEP;
+    vc->gfx.scale_y += VC_SCALE_STEP;
 
-    gd_update_windowsize(s);
+    gd_update_windowsize(vc);
 }
 
 static void gd_menu_zoom_out(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
     gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->zoom_fit_item),
                                    FALSE);
 
-    s->scale_x -= .25;
-    s->scale_y -= .25;
+    vc->gfx.scale_x -= VC_SCALE_STEP;
+    vc->gfx.scale_y -= VC_SCALE_STEP;
 
-    s->scale_x = MAX(s->scale_x, .25);
-    s->scale_y = MAX(s->scale_y, .25);
+    vc->gfx.scale_x = MAX(vc->gfx.scale_x, VC_SCALE_MIN);
+    vc->gfx.scale_y = MAX(vc->gfx.scale_y, VC_SCALE_MIN);
 
-    gd_update_windowsize(s);
+    gd_update_windowsize(vc);
 }
 
 static void gd_menu_zoom_fixed(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
-    s->scale_x = 1.0;
-    s->scale_y = 1.0;
+    vc->gfx.scale_x = 1.0;
+    vc->gfx.scale_y = 1.0;
 
-    gd_update_windowsize(s);
+    gd_update_windowsize(vc);
 }
 
 static void gd_menu_zoom_fit(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
     if (gtk_check_menu_item_get_active(GTK_CHECK_MENU_ITEM(s->zoom_fit_item))) {
         s->free_scale = TRUE;
     } else {
         s->free_scale = FALSE;
-        s->scale_x = 1.0;
-        s->scale_y = 1.0;
-        gd_update_windowsize(s);
+        vc->gfx.scale_x = 1.0;
+        vc->gfx.scale_y = 1.0;
     }
 
-    gd_update_full_redraw(s);
+    gd_update_windowsize(vc);
+    gd_update_full_redraw(vc);
 }
 
-static void gd_grab_keyboard(GtkDisplayState *s)
+static void gd_grab_keyboard(VirtualConsole *vc)
 {
 #if GTK_CHECK_VERSION(3, 0, 0)
-    GdkDisplay *display = gtk_widget_get_display(s->drawing_area);
+    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     GList *devices = gdk_device_manager_list_devices(mgr,
                                                      GDK_DEVICE_TYPE_MASTER);
@@ -927,7 +1153,7 @@
         GdkDevice *dev = tmp->data;
         if (gdk_device_get_source(dev) == GDK_SOURCE_KEYBOARD) {
             gdk_device_grab(dev,
-                            gtk_widget_get_window(s->drawing_area),
+                            gtk_widget_get_window(vc->gfx.drawing_area),
                             GDK_OWNERSHIP_NONE,
                             FALSE,
                             GDK_KEY_PRESS_MASK | GDK_KEY_RELEASE_MASK,
@@ -938,16 +1164,25 @@
     }
     g_list_free(devices);
 #else
-    gdk_keyboard_grab(gtk_widget_get_window(s->drawing_area),
+    gdk_keyboard_grab(gtk_widget_get_window(vc->gfx.drawing_area),
                       FALSE,
                       GDK_CURRENT_TIME);
 #endif
+    vc->s->kbd_owner = vc;
+    trace_gd_grab(vc->label, "kbd", true);
 }
 
 static void gd_ungrab_keyboard(GtkDisplayState *s)
 {
+    VirtualConsole *vc = s->kbd_owner;
+
+    if (vc == NULL) {
+        return;
+    }
+    s->kbd_owner = NULL;
+
 #if GTK_CHECK_VERSION(3, 0, 0)
-    GdkDisplay *display = gtk_widget_get_display(s->drawing_area);
+    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     GList *devices = gdk_device_manager_list_devices(mgr,
                                                      GDK_DEVICE_TYPE_MASTER);
@@ -964,11 +1199,12 @@
 #else
     gdk_keyboard_ungrab(GDK_CURRENT_TIME);
 #endif
+    trace_gd_grab(vc->label, "kbd", false);
 }
 
-static void gd_grab_pointer(GtkDisplayState *s)
+static void gd_grab_pointer(VirtualConsole *vc)
 {
-    GdkDisplay *display = gtk_widget_get_display(s->drawing_area);
+    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
 #if GTK_CHECK_VERSION(3, 0, 0)
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     GList *devices = gdk_device_manager_list_devices(mgr,
@@ -978,7 +1214,7 @@
         GdkDevice *dev = tmp->data;
         if (gdk_device_get_source(dev) == GDK_SOURCE_MOUSE) {
             gdk_device_grab(dev,
-                            gtk_widget_get_window(s->drawing_area),
+                            gtk_widget_get_window(vc->gfx.drawing_area),
                             GDK_OWNERSHIP_NONE,
                             FALSE, /* All events to come to our
                                       window directly */
@@ -987,16 +1223,16 @@
                             GDK_BUTTON_RELEASE_MASK |
                             GDK_BUTTON_MOTION_MASK |
                             GDK_SCROLL_MASK,
-                            s->null_cursor,
+                            vc->s->null_cursor,
                             GDK_CURRENT_TIME);
         }
         tmp = tmp->next;
     }
     g_list_free(devices);
     gdk_device_get_position(gdk_device_manager_get_client_pointer(mgr),
-                            NULL, &s->grab_x_root, &s->grab_y_root);
+                            NULL, &vc->s->grab_x_root, &vc->s->grab_y_root);
 #else
-    gdk_pointer_grab(gtk_widget_get_window(s->drawing_area),
+    gdk_pointer_grab(gtk_widget_get_window(vc->gfx.drawing_area),
                      FALSE, /* All events to come to our window directly */
                      GDK_POINTER_MOTION_MASK |
                      GDK_BUTTON_PRESS_MASK |
@@ -1004,16 +1240,25 @@
                      GDK_BUTTON_MOTION_MASK |
                      GDK_SCROLL_MASK,
                      NULL, /* Allow cursor to move over entire desktop */
-                     s->null_cursor,
+                     vc->s->null_cursor,
                      GDK_CURRENT_TIME);
     gdk_display_get_pointer(display, NULL,
-                            &s->grab_x_root, &s->grab_y_root, NULL);
+                            &vc->s->grab_x_root, &vc->s->grab_y_root, NULL);
 #endif
+    vc->s->ptr_owner = vc;
+    trace_gd_grab(vc->label, "ptr", true);
 }
 
 static void gd_ungrab_pointer(GtkDisplayState *s)
 {
-    GdkDisplay *display = gtk_widget_get_display(s->drawing_area);
+    VirtualConsole *vc = s->ptr_owner;
+
+    if (vc == NULL) {
+        return;
+    }
+    s->ptr_owner = NULL;
+
+    GdkDisplay *display = gtk_widget_get_display(vc->gfx.drawing_area);
 #if GTK_CHECK_VERSION(3, 0, 0)
     GdkDeviceManager *mgr = gdk_display_get_device_manager(display);
     GList *devices = gdk_device_manager_list_devices(mgr,
@@ -1029,51 +1274,65 @@
     }
     g_list_free(devices);
     gdk_device_warp(gdk_device_manager_get_client_pointer(mgr),
-                    gtk_widget_get_screen(s->drawing_area),
-                    s->grab_x_root, s->grab_y_root);
+                    gtk_widget_get_screen(vc->gfx.drawing_area),
+                    vc->s->grab_x_root, vc->s->grab_y_root);
 #else
     gdk_pointer_ungrab(GDK_CURRENT_TIME);
     gdk_display_warp_pointer(display,
-                             gtk_widget_get_screen(s->drawing_area),
-                             s->grab_x_root, s->grab_y_root);
+                             gtk_widget_get_screen(vc->gfx.drawing_area),
+                             vc->s->grab_x_root, vc->s->grab_y_root);
 #endif
+    trace_gd_grab(vc->label, "ptr", false);
 }
 
 static void gd_menu_grab_input(GtkMenuItem *item, void *opaque)
 {
     GtkDisplayState *s = opaque;
+    VirtualConsole *vc = gd_vc_find_current(s);
 
     if (gd_is_grab_active(s)) {
-        gd_grab_keyboard(s);
-        gd_grab_pointer(s);
+        if (!gd_grab_on_hover(s)) {
+            gd_grab_keyboard(vc);
+        }
+        gd_grab_pointer(vc);
     } else {
         gd_ungrab_keyboard(s);
         gd_ungrab_pointer(s);
     }
 
     gd_update_caption(s);
-    gd_update_cursor(s, FALSE);
+    gd_update_cursor(vc);
 }
 
 static void gd_change_page(GtkNotebook *nb, gpointer arg1, guint arg2,
                            gpointer data)
 {
     GtkDisplayState *s = data;
-    guint last_page;
+    VirtualConsole *vc;
     gboolean on_vga;
 
     if (!gtk_widget_get_realized(s->notebook)) {
         return;
     }
 
-    last_page = gtk_notebook_get_current_page(nb);
-
-    if (last_page) {
-        gtk_widget_set_size_request(s->vc[last_page - 1].terminal, -1, -1);
+#ifdef VTE_RESIZE_HACK
+    vc = gd_vc_find_current(s);
+    if (vc && vc->type == GD_VC_VTE) {
+        gtk_widget_hide(vc->vte.terminal);
     }
-
-    on_vga = arg2 == 0;
-
+#endif
+    vc = gd_vc_find_by_page(s, arg2);
+    if (!vc) {
+        return;
+    }
+#ifdef VTE_RESIZE_HACK
+    if (vc->type == GD_VC_VTE) {
+        gtk_widget_show(vc->vte.terminal);
+    }
+#endif
+    gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(vc->menu_item),
+                                   TRUE);
+    on_vga = (vc->type == GD_VC_GFX);
     if (!on_vga) {
         gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                        FALSE);
@@ -1081,71 +1340,88 @@
         gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->grab_item),
                                        TRUE);
     }
-
-    if (arg2 == 0) {
-        gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(s->vga_item), TRUE);
-    } else {
-#if defined(CONFIG_VTE)
-        VirtualConsole *vc = &s->vc[arg2 - 1];
-        VteTerminal *term = VTE_TERMINAL(vc->terminal);
-        int width, height;
-
-        width = 80 * vte_terminal_get_char_width(term);
-        height = 25 * vte_terminal_get_char_height(term);
-
-        gtk_check_menu_item_set_active(GTK_CHECK_MENU_ITEM(vc->menu_item), TRUE);
-        gtk_widget_set_size_request(vc->terminal, width, height);
-#else
-        g_assert_not_reached();
-#endif
-    }
-
     gtk_widget_set_sensitive(s->grab_item, on_vga);
 
-    gd_update_cursor(s, TRUE);
+    gd_update_windowsize(vc);
+    gd_update_cursor(vc);
 }
 
-static gboolean gd_enter_event(GtkWidget *widget, GdkEventCrossing *crossing, gpointer data)
+static gboolean gd_enter_event(GtkWidget *widget, GdkEventCrossing *crossing,
+                               gpointer opaque)
 {
-    GtkDisplayState *s = data;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
 
-    if (!gd_is_grab_active(s) && gd_grab_on_hover(s)) {
-        gd_grab_keyboard(s);
+    if (gd_grab_on_hover(s)) {
+        gd_ungrab_keyboard(s);
+        gd_grab_keyboard(vc);
+        gd_update_caption(s);
     }
-
     return TRUE;
 }
 
-static gboolean gd_leave_event(GtkWidget *widget, GdkEventCrossing *crossing, gpointer data)
+static gboolean gd_leave_event(GtkWidget *widget, GdkEventCrossing *crossing,
+                               gpointer opaque)
 {
-    GtkDisplayState *s = data;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
 
-    if (!gd_is_grab_active(s) && gd_grab_on_hover(s)) {
+    if (gd_grab_on_hover(s)) {
         gd_ungrab_keyboard(s);
+        gd_update_caption(s);
     }
-
     return TRUE;
 }
 
 static gboolean gd_focus_out_event(GtkWidget *widget,
-                                   GdkEventCrossing *crossing, gpointer data)
+                                   GdkEventCrossing *crossing, gpointer opaque)
 {
-    GtkDisplayState *s = data;
+    VirtualConsole *vc = opaque;
+    GtkDisplayState *s = vc->s;
 
     gtk_release_modifiers(s);
-
     return TRUE;
 }
 
 /** Virtual Console Callbacks **/
 
+static GSList *gd_vc_menu_init(GtkDisplayState *s, VirtualConsole *vc,
+                               int idx, GSList *group, GtkWidget *view_menu)
+{
+    char path[32];
+
+    snprintf(path, sizeof(path), "<QEMU>/View/VC%d", idx);
+
+    vc->menu_item = gtk_radio_menu_item_new_with_mnemonic(group, vc->label);
+    group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(vc->menu_item));
+    gtk_menu_item_set_accel_path(GTK_MENU_ITEM(vc->menu_item), path);
+    gtk_accel_map_add_entry(path, GDK_KEY_1 + idx, HOTKEY_MODIFIERS);
+
+    g_signal_connect(vc->menu_item, "activate",
+                     G_CALLBACK(gd_menu_switch_vc), s);
+    gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), vc->menu_item);
+
+    return group;
+}
+
+#if defined(CONFIG_VTE)
+static void gd_vc_adjustment_changed(GtkAdjustment *adjustment, void *opaque)
+{
+    VirtualConsole *vc = opaque;
+
+    if (gtk_adjustment_get_upper(adjustment) >
+        gtk_adjustment_get_page_size(adjustment)) {
+        gtk_widget_show(vc->vte.scrollbar);
+    } else {
+        gtk_widget_hide(vc->vte.scrollbar);
+    }
+}
+
 static int gd_vc_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
-#if defined(CONFIG_VTE)
     VirtualConsole *vc = chr->opaque;
 
-    vte_terminal_feed(VTE_TERMINAL(vc->terminal), (const char *)buf, len);
-#endif
+    vte_terminal_feed(VTE_TERMINAL(vc->vte.terminal), (const char *)buf, len);
     return len;
 }
 
@@ -1166,115 +1442,132 @@
     return chr;
 }
 
-void early_gtk_display_init(void)
-{
-    register_vc_handler(gd_vc_handler);
-}
-
-#if defined(CONFIG_VTE)
 static gboolean gd_vc_in(VteTerminal *terminal, gchar *text, guint size,
                          gpointer user_data)
 {
     VirtualConsole *vc = user_data;
 
-    qemu_chr_be_write(vc->chr, (uint8_t  *)text, (unsigned int)size);
+    qemu_chr_be_write(vc->vte.chr, (uint8_t  *)text, (unsigned int)size);
     return TRUE;
 }
-#endif
 
-static GSList *gd_vc_init(GtkDisplayState *s, VirtualConsole *vc, int index, GSList *group,
-                          GtkWidget *view_menu)
+static GSList *gd_vc_vte_init(GtkDisplayState *s, VirtualConsole *vc,
+                              CharDriverState *chr, int idx,
+                              GSList *group, GtkWidget *view_menu)
 {
-#if defined(CONFIG_VTE)
-    const char *label;
     char buffer[32];
-    char path[32];
-    GtkWidget *scrolled_window;
+    GtkWidget *box;
+    GtkWidget *scrollbar;
     GtkAdjustment *vadjustment;
 
-    snprintf(buffer, sizeof(buffer), "vc%d", index);
-    snprintf(path, sizeof(path), "<QEMU>/View/VC%d", index);
+    vc->s = s;
+    vc->vte.chr = chr;
 
-    vc->chr = vcs[index];
+    snprintf(buffer, sizeof(buffer), "vc%d", idx);
+    vc->label = g_strdup_printf("%s", vc->vte.chr->label
+                                ? vc->vte.chr->label : buffer);
+    group = gd_vc_menu_init(s, vc, idx, group, view_menu);
 
-    if (vc->chr->label) {
-        label = vc->chr->label;
-    } else {
-        label = buffer;
-    }
+    vc->vte.terminal = vte_terminal_new();
+    g_signal_connect(vc->vte.terminal, "commit", G_CALLBACK(gd_vc_in), vc);
 
-    vc->menu_item = gtk_radio_menu_item_new_with_mnemonic(group, label);
-    group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(vc->menu_item));
-    gtk_menu_item_set_accel_path(GTK_MENU_ITEM(vc->menu_item), path);
-    gtk_accel_map_add_entry(path, GDK_KEY_2 + index, HOTKEY_MODIFIERS);
-
-    vc->terminal = vte_terminal_new();
-    g_signal_connect(vc->terminal, "commit", G_CALLBACK(gd_vc_in), vc);
-
-    vte_terminal_set_scrollback_lines(VTE_TERMINAL(vc->terminal), -1);
+    vte_terminal_set_scrollback_lines(VTE_TERMINAL(vc->vte.terminal), -1);
+    vte_terminal_set_size(VTE_TERMINAL(vc->vte.terminal),
+                          VC_TERM_X_MIN, VC_TERM_Y_MIN);
 
 #if VTE_CHECK_VERSION(0, 28, 0) && GTK_CHECK_VERSION(3, 0, 0)
-    vadjustment = gtk_scrollable_get_vadjustment(GTK_SCROLLABLE(vc->terminal));
+    vadjustment = gtk_scrollable_get_vadjustment
+        (GTK_SCROLLABLE(vc->vte.terminal));
 #else
-    vadjustment = vte_terminal_get_adjustment(VTE_TERMINAL(vc->terminal));
+    vadjustment = vte_terminal_get_adjustment(VTE_TERMINAL(vc->vte.terminal));
 #endif
 
-    scrolled_window = gtk_scrolled_window_new(NULL, vadjustment);
-    gtk_container_add(GTK_CONTAINER(scrolled_window), vc->terminal);
+#if GTK_CHECK_VERSION(3, 0, 0)
+    box = gtk_box_new(GTK_ORIENTATION_HORIZONTAL, 2);
+    scrollbar = gtk_scrollbar_new(GTK_ORIENTATION_VERTICAL, vadjustment);
+#else
+    box = gtk_hbox_new(false, 2);
+    scrollbar = gtk_vscrollbar_new(vadjustment);
+#endif
 
-    vte_terminal_set_size(VTE_TERMINAL(vc->terminal), 80, 25);
+    gtk_box_pack_start(GTK_BOX(box), vc->vte.terminal, TRUE, TRUE, 0);
+    gtk_box_pack_start(GTK_BOX(box), scrollbar, FALSE, FALSE, 0);
 
-    vc->chr->opaque = vc;
-    vc->scrolled_window = scrolled_window;
+    vc->vte.chr->opaque = vc;
+    vc->vte.box = box;
+    vc->vte.scrollbar = scrollbar;
 
-    gtk_scrolled_window_set_policy(GTK_SCROLLED_WINDOW(vc->scrolled_window),
-                                   GTK_POLICY_AUTOMATIC, GTK_POLICY_AUTOMATIC);
+    g_signal_connect(vadjustment, "changed",
+                     G_CALLBACK(gd_vc_adjustment_changed), vc);
 
-    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook), scrolled_window, gtk_label_new(label));
-    g_signal_connect(vc->menu_item, "activate",
-                     G_CALLBACK(gd_menu_switch_vc), s);
+    vc->type = GD_VC_VTE;
+    vc->tab_item = box;
+    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook), vc->tab_item,
+                             gtk_label_new(vc->label));
 
-    gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), vc->menu_item);
-
-    qemu_chr_be_generic_open(vc->chr);
-    if (vc->chr->init) {
-        vc->chr->init(vc->chr);
+    qemu_chr_be_generic_open(vc->vte.chr);
+    if (vc->vte.chr->init) {
+        vc->vte.chr->init(vc->vte.chr);
     }
 
-#endif /* CONFIG_VTE */
     return group;
 }
 
+static void gd_vcs_init(GtkDisplayState *s, GSList *group,
+                        GtkWidget *view_menu)
+{
+    int i;
+
+    for (i = 0; i < nb_vcs; i++) {
+        VirtualConsole *vc = &s->vc[s->nb_vcs];
+        group = gd_vc_vte_init(s, vc, vcs[i], s->nb_vcs, group, view_menu);
+        s->nb_vcs++;
+    }
+}
+#endif /* CONFIG_VTE */
+
 /** Window Creation **/
 
+static void gd_connect_vc_gfx_signals(VirtualConsole *vc)
+{
+#if GTK_CHECK_VERSION(3, 0, 0)
+    g_signal_connect(vc->gfx.drawing_area, "draw",
+                     G_CALLBACK(gd_draw_event), vc);
+#else
+    g_signal_connect(vc->gfx.drawing_area, "expose-event",
+                     G_CALLBACK(gd_expose_event), vc);
+#endif
+    g_signal_connect(vc->gfx.drawing_area, "event",
+                     G_CALLBACK(gd_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "button-press-event",
+                     G_CALLBACK(gd_button_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "button-release-event",
+                     G_CALLBACK(gd_button_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "scroll-event",
+                     G_CALLBACK(gd_scroll_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "key-press-event",
+                     G_CALLBACK(gd_key_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "key-release-event",
+                     G_CALLBACK(gd_key_event), vc);
+
+    g_signal_connect(vc->gfx.drawing_area, "enter-notify-event",
+                     G_CALLBACK(gd_enter_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "leave-notify-event",
+                     G_CALLBACK(gd_leave_event), vc);
+    g_signal_connect(vc->gfx.drawing_area, "focus-out-event",
+                     G_CALLBACK(gd_focus_out_event), vc);
+}
+
 static void gd_connect_signals(GtkDisplayState *s)
 {
     g_signal_connect(s->show_tabs_item, "activate",
                      G_CALLBACK(gd_menu_show_tabs), s);
+    g_signal_connect(s->untabify_item, "activate",
+                     G_CALLBACK(gd_menu_untabify), s);
 
     g_signal_connect(s->window, "delete-event",
                      G_CALLBACK(gd_window_close), s);
 
-#if GTK_CHECK_VERSION(3, 0, 0)
-    g_signal_connect(s->drawing_area, "draw",
-                     G_CALLBACK(gd_draw_event), s);
-#else
-    g_signal_connect(s->drawing_area, "expose-event",
-                     G_CALLBACK(gd_expose_event), s);
-#endif
-    g_signal_connect(s->drawing_area, "event",
-                     G_CALLBACK(gd_event), s);
-    g_signal_connect(s->drawing_area, "button-press-event",
-                     G_CALLBACK(gd_button_event), s);
-    g_signal_connect(s->drawing_area, "button-release-event",
-                     G_CALLBACK(gd_button_event), s);
-    g_signal_connect(s->drawing_area, "scroll-event",
-                     G_CALLBACK(gd_scroll_event), s);
-    g_signal_connect(s->drawing_area, "key-press-event",
-                     G_CALLBACK(gd_key_event), s);
-    g_signal_connect(s->drawing_area, "key-release-event",
-                     G_CALLBACK(gd_key_event), s);
-
     g_signal_connect(s->pause_item, "activate",
                      G_CALLBACK(gd_menu_pause), s);
     g_signal_connect(s->reset_item, "activate",
@@ -1293,18 +1586,10 @@
                      G_CALLBACK(gd_menu_zoom_fixed), s);
     g_signal_connect(s->zoom_fit_item, "activate",
                      G_CALLBACK(gd_menu_zoom_fit), s);
-    g_signal_connect(s->vga_item, "activate",
-                     G_CALLBACK(gd_menu_switch_vc), s);
     g_signal_connect(s->grab_item, "activate",
                      G_CALLBACK(gd_menu_grab_input), s);
     g_signal_connect(s->notebook, "switch-page",
                      G_CALLBACK(gd_change_page), s);
-    g_signal_connect(s->drawing_area, "enter-notify-event",
-                     G_CALLBACK(gd_enter_event), s);
-    g_signal_connect(s->drawing_area, "leave-notify-event",
-                     G_CALLBACK(gd_leave_event), s);
-    g_signal_connect(s->drawing_area, "focus-out-event",
-                     G_CALLBACK(gd_focus_out_event), s);
 }
 
 static GtkWidget *gd_create_menu_machine(GtkDisplayState *s, GtkAccelGroup *accel_group)
@@ -1340,12 +1625,68 @@
     return machine_menu;
 }
 
+static const DisplayChangeListenerOps dcl_ops = {
+    .dpy_name          = "gtk",
+    .dpy_gfx_update    = gd_update,
+    .dpy_gfx_switch    = gd_switch,
+    .dpy_refresh       = gd_refresh,
+    .dpy_mouse_set     = gd_mouse_set,
+    .dpy_cursor_define = gd_cursor_define,
+};
+
+static GSList *gd_vc_gfx_init(GtkDisplayState *s, VirtualConsole *vc,
+                              QemuConsole *con, int idx,
+                              GSList *group, GtkWidget *view_menu)
+{
+    Error *local_err = NULL;
+    Object *obj;
+
+    obj = object_property_get_link(OBJECT(con), "device", &local_err);
+    if (obj) {
+        vc->label = g_strdup_printf("%s", object_get_typename(obj));
+    } else {
+        vc->label = g_strdup_printf("VGA");
+    }
+
+    vc->s = s;
+    vc->gfx.scale_x = 1.0;
+    vc->gfx.scale_y = 1.0;
+
+    vc->gfx.drawing_area = gtk_drawing_area_new();
+    gtk_widget_add_events(vc->gfx.drawing_area,
+                          GDK_POINTER_MOTION_MASK |
+                          GDK_BUTTON_PRESS_MASK |
+                          GDK_BUTTON_RELEASE_MASK |
+                          GDK_BUTTON_MOTION_MASK |
+                          GDK_ENTER_NOTIFY_MASK |
+                          GDK_LEAVE_NOTIFY_MASK |
+                          GDK_SCROLL_MASK |
+                          GDK_KEY_PRESS_MASK);
+    gtk_widget_set_double_buffered(vc->gfx.drawing_area, FALSE);
+    gtk_widget_set_can_focus(vc->gfx.drawing_area, TRUE);
+
+    vc->type = GD_VC_GFX;
+    vc->tab_item = vc->gfx.drawing_area;
+    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook),
+                             vc->tab_item, gtk_label_new(vc->label));
+    gd_connect_vc_gfx_signals(vc);
+
+    group = gd_vc_menu_init(s, vc, idx, group, view_menu);
+
+    vc->gfx.dcl.ops = &dcl_ops;
+    vc->gfx.dcl.con = con;
+    register_displaychangelistener(&vc->gfx.dcl);
+
+    return group;
+}
+
 static GtkWidget *gd_create_menu_view(GtkDisplayState *s, GtkAccelGroup *accel_group)
 {
     GSList *group = NULL;
     GtkWidget *view_menu;
     GtkWidget *separator;
-    int i;
+    QemuConsole *con;
+    int vc;
 
     view_menu = gtk_menu_new();
     gtk_menu_set_accel_group(GTK_MENU(view_menu), accel_group);
@@ -1400,26 +1741,31 @@
     separator = gtk_separator_menu_item_new();
     gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), separator);
 
-    s->vga_item = gtk_radio_menu_item_new_with_mnemonic(group, "_VGA");
-    group = gtk_radio_menu_item_get_group(GTK_RADIO_MENU_ITEM(s->vga_item));
-    gtk_menu_item_set_accel_path(GTK_MENU_ITEM(s->vga_item),
-                                 "<QEMU>/View/VGA");
-    gtk_accel_map_add_entry("<QEMU>/View/VGA", GDK_KEY_1, HOTKEY_MODIFIERS);
-    gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), s->vga_item);
-
-    for (i = 0; i < nb_vcs; i++) {
-        VirtualConsole *vc = &s->vc[i];
-
-        group = gd_vc_init(s, vc, i, group, view_menu);
+    /* gfx */
+    for (vc = 0;; vc++) {
+        con = qemu_console_lookup_by_index(vc);
+        if (!con || !qemu_console_is_graphic(con)) {
+            break;
+        }
+        group = gd_vc_gfx_init(s, &s->vc[vc], con,
+                               vc, group, view_menu);
         s->nb_vcs++;
     }
 
+#if defined(CONFIG_VTE)
+    /* vte */
+    gd_vcs_init(s, group, view_menu);
+#endif
+
     separator = gtk_separator_menu_item_new();
     gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), separator);
 
     s->show_tabs_item = gtk_check_menu_item_new_with_mnemonic(_("Show _Tabs"));
     gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), s->show_tabs_item);
 
+    s->untabify_item = gtk_menu_item_new_with_mnemonic(_("Detach Tab"));
+    gtk_menu_shell_append(GTK_MENU_SHELL(view_menu), s->untabify_item);
+
     return view_menu;
 }
 
@@ -1445,14 +1791,28 @@
     s->accel_group = accel_group;
 }
 
-static const DisplayChangeListenerOps dcl_ops = {
-    .dpy_name          = "gtk",
-    .dpy_gfx_update    = gd_update,
-    .dpy_gfx_switch    = gd_switch,
-    .dpy_refresh       = gd_refresh,
-    .dpy_mouse_set     = gd_mouse_set,
-    .dpy_cursor_define = gd_cursor_define,
-};
+static void gd_set_keycode_type(GtkDisplayState *s)
+{
+#ifndef _WIN32
+    char *keycodes = NULL;
+    GdkDisplay *display = gtk_widget_get_display(s->window);
+    Display *x11_display = gdk_x11_display_get_xdisplay(display);
+    XkbDescPtr desc = XkbGetKeyboard(x11_display, XkbGBN_AllComponentsMask,
+                                     XkbUseCoreKbd);
+
+    if (desc && desc->names) {
+        keycodes = XGetAtomName(x11_display, desc->names->keycodes);
+    }
+    if (keycodes == NULL) {
+        fprintf(stderr, "could not lookup keycode name\n");
+    } else if (strstart(keycodes, "evdev", NULL)) {
+        s->has_evdev = true;
+    } else if (!strstart(keycodes, "xfree86", NULL)) {
+        fprintf(stderr, "unknown keycodes `%s', please report to "
+                "qemu-devel@nongnu.org\n", keycodes);
+    }
+#endif
+}
 
 void gtk_display_init(DisplayState *ds, bool full_screen, bool grab_on_hover)
 {
@@ -1461,9 +1821,6 @@
 
     gtk_init(NULL, NULL);
 
-    s->dcl.ops = &dcl_ops;
-    s->dcl.con = qemu_console_lookup_by_index(0);
-
     s->window = gtk_window_new(GTK_WINDOW_TOPLEVEL);
 #if GTK_CHECK_VERSION(3, 2, 0)
     s->vbox = gtk_box_new(GTK_ORIENTATION_VERTICAL, 0);
@@ -1471,11 +1828,8 @@
     s->vbox = gtk_vbox_new(FALSE, 0);
 #endif
     s->notebook = gtk_notebook_new();
-    s->drawing_area = gtk_drawing_area_new();
     s->menu_bar = gtk_menu_bar_new();
 
-    s->scale_x = 1.0;
-    s->scale_y = 1.0;
     s->free_scale = FALSE;
 
     setlocale(LC_ALL, "");
@@ -1488,8 +1842,6 @@
     qemu_add_mouse_mode_change_notifier(&s->mouse_mode_notifier);
     qemu_add_vm_change_state_handler(gd_change_runstate, s);
 
-    gtk_notebook_append_page(GTK_NOTEBOOK(s->notebook), s->drawing_area, gtk_label_new("VGA"));
-
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "qemu_logo_no_text.svg");
     if (filename) {
         GError *error = NULL;
@@ -1506,18 +1858,6 @@
 
     gd_connect_signals(s);
 
-    gtk_widget_add_events(s->drawing_area,
-                          GDK_POINTER_MOTION_MASK |
-                          GDK_BUTTON_PRESS_MASK |
-                          GDK_BUTTON_RELEASE_MASK |
-                          GDK_BUTTON_MOTION_MASK |
-                          GDK_ENTER_NOTIFY_MASK |
-                          GDK_LEAVE_NOTIFY_MASK |
-                          GDK_SCROLL_MASK |
-                          GDK_KEY_PRESS_MASK);
-    gtk_widget_set_double_buffered(s->drawing_area, FALSE);
-    gtk_widget_set_can_focus(s->drawing_area, TRUE);
-
     gtk_notebook_set_show_tabs(GTK_NOTEBOOK(s->notebook), FALSE);
     gtk_notebook_set_show_border(GTK_NOTEBOOK(s->notebook), FALSE);
 
@@ -1530,6 +1870,21 @@
 
     gtk_widget_show_all(s->window);
 
+#ifdef VTE_RESIZE_HACK
+    {
+        VirtualConsole *cur = gd_vc_find_current(s);
+        int i;
+
+        for (i = 0; i < s->nb_vcs; i++) {
+            VirtualConsole *vc = &s->vc[i];
+            if (vc && vc->type == GD_VC_VTE && vc != cur) {
+                gtk_widget_hide(vc->vte.terminal);
+            }
+        }
+        gd_update_windowsize(cur);
+    }
+#endif
+
     if (full_screen) {
         gtk_menu_item_activate(GTK_MENU_ITEM(s->full_screen_item));
     }
@@ -1537,7 +1892,12 @@
         gtk_menu_item_activate(GTK_MENU_ITEM(s->grab_on_hover_item));
     }
 
-    register_displaychangelistener(&s->dcl);
+    gd_set_keycode_type(s);
+}
 
-    global_state = s;
+void early_gtk_display_init(void)
+{
+#if defined(CONFIG_VTE)
+    register_vc_handler(gd_vc_handler);
+#endif
 }
diff --git a/ui/input-keymap.c b/ui/input-keymap.c
index 6da4495..5d29935 100644
--- a/ui/input-keymap.c
+++ b/ui/input-keymap.c
@@ -13,6 +13,8 @@
     [Q_KEY_CODE_CTRL] = 0x1d,
     [Q_KEY_CODE_CTRL_R] = 0x9d,
 
+    [Q_KEY_CODE_META_L] = 0xdb,
+    [Q_KEY_CODE_META_R] = 0xdc,
     [Q_KEY_CODE_MENU] = 0xdd,
 
     [Q_KEY_CODE_ESC] = 0x01,
@@ -129,7 +131,7 @@
     [Q_KEY_CODE_MAX] = 0,
 };
 
-static int number_to_qcode[0xff];
+static int number_to_qcode[0x100];
 
 int qemu_input_key_value_to_number(const KeyValue *value)
 {
@@ -141,7 +143,7 @@
     }
 }
 
-int qemu_input_key_value_to_qcode(const KeyValue *value)
+int qemu_input_key_number_to_qcode(uint8_t nr)
 {
     static int first = true;
 
@@ -155,11 +157,16 @@
         }
     }
 
+    return number_to_qcode[nr];
+}
+
+int qemu_input_key_value_to_qcode(const KeyValue *value)
+{
     if (value->kind == KEY_VALUE_KIND_QCODE) {
         return value->qcode;
     } else {
         assert(value->kind == KEY_VALUE_KIND_NUMBER);
-        return number_to_qcode[value->number];
+        return qemu_input_key_number_to_qcode(value->number);
     }
 }
 
diff --git a/ui/input-legacy.c b/ui/input-legacy.c
index 2a53860..3025f50 100644
--- a/ui/input-legacy.c
+++ b/ui/input-legacy.c
@@ -74,27 +74,6 @@
     return i;
 }
 
-static KeyValue **keyvalues;
-static int keyvalues_size;
-static QEMUTimer *key_timer;
-
-static void free_keyvalues(void)
-{
-    g_free(keyvalues);
-    keyvalues = NULL;
-    keyvalues_size = 0;
-}
-
-static void release_keys(void *opaque)
-{
-    while (keyvalues_size > 0) {
-        qemu_input_event_send_key(NULL, keyvalues[--keyvalues_size],
-                                  false);
-    }
-
-    free_keyvalues();
-}
-
 static KeyValue *copy_key_value(KeyValue *src)
 {
     KeyValue *dst = g_new(KeyValue, 1);
@@ -107,30 +86,18 @@
 {
     KeyValueList *p;
 
-    if (!key_timer) {
-        key_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, release_keys, NULL);
-    }
-
-    if (keyvalues != NULL) {
-        timer_del(key_timer);
-        release_keys(NULL);
-    }
-
     if (!has_hold_time) {
-        hold_time = 100;
+        hold_time = 0; /* use default */
     }
 
     for (p = keys; p != NULL; p = p->next) {
         qemu_input_event_send_key(NULL, copy_key_value(p->value), true);
-
-        keyvalues = g_realloc(keyvalues, sizeof(KeyValue *) *
-                              (keyvalues_size + 1));
-        keyvalues[keyvalues_size++] = copy_key_value(p->value);
+        qemu_input_event_send_key_delay(hold_time);
     }
-
-    /* delayed key up events */
-    timer_mod(key_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
-              muldiv64(get_ticks_per_sec(), hold_time, 1000));
+    for (p = keys; p != NULL; p = p->next) {
+        qemu_input_event_send_key(NULL, copy_key_value(p->value), false);
+        qemu_input_event_send_key_delay(hold_time);
+    }
 }
 
 static void legacy_kbd_event(DeviceState *dev, QemuConsole *src,
diff --git a/ui/input.c b/ui/input.c
index fc91fba..89d9db7 100644
--- a/ui/input.c
+++ b/ui/input.c
@@ -1,3 +1,4 @@
+#include "hw/qdev.h"
 #include "sysemu/sysemu.h"
 #include "qapi-types.h"
 #include "qmp-commands.h"
@@ -10,13 +11,34 @@
     QemuInputHandler  *handler;
     int               id;
     int               events;
+    QemuConsole       *con;
     QTAILQ_ENTRY(QemuInputHandlerState) node;
 };
+
+typedef struct QemuInputEventQueue QemuInputEventQueue;
+struct QemuInputEventQueue {
+    enum {
+        QEMU_INPUT_QUEUE_DELAY = 1,
+        QEMU_INPUT_QUEUE_EVENT,
+        QEMU_INPUT_QUEUE_SYNC,
+    } type;
+    QEMUTimer *timer;
+    uint32_t delay_ms;
+    QemuConsole *src;
+    InputEvent *evt;
+    QTAILQ_ENTRY(QemuInputEventQueue) node;
+};
+
 static QTAILQ_HEAD(, QemuInputHandlerState) handlers =
     QTAILQ_HEAD_INITIALIZER(handlers);
 static NotifierList mouse_mode_notifiers =
     NOTIFIER_LIST_INITIALIZER(mouse_mode_notifiers);
 
+static QTAILQ_HEAD(QemuInputEventQueueHead, QemuInputEventQueue) kbd_queue =
+    QTAILQ_HEAD_INITIALIZER(kbd_queue);
+static QEMUTimer *kbd_timer;
+static uint32_t kbd_default_delay_ms = 10;
+
 QemuInputHandlerState *qemu_input_handler_register(DeviceState *dev,
                                                    QemuInputHandler *handler)
 {
@@ -53,12 +75,46 @@
     qemu_input_check_mode_change();
 }
 
+void qemu_input_handler_bind(QemuInputHandlerState *s,
+                             const char *device_id, int head,
+                             Error **errp)
+{
+    DeviceState *dev;
+    QemuConsole *con;
+
+    dev = qdev_find_recursive(sysbus_get_default(), device_id);
+    if (dev == NULL) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device_id);
+        return;
+    }
+
+    con = qemu_console_lookup_by_device(dev, head);
+    if (con == NULL) {
+        error_setg(errp, "Device %s is not bound to a QemuConsole", device_id);
+        return;
+    }
+
+    s->con = con;
+}
+
 static QemuInputHandlerState*
-qemu_input_find_handler(uint32_t mask)
+qemu_input_find_handler(uint32_t mask, QemuConsole *con)
 {
     QemuInputHandlerState *s;
 
     QTAILQ_FOREACH(s, &handlers, node) {
+        if (s->con == NULL || s->con != con) {
+            continue;
+        }
+        if (mask & s->handler->mask) {
+            return s;
+        }
+    }
+
+    QTAILQ_FOREACH(s, &handlers, node) {
+        if (s->con != NULL) {
+            continue;
+        }
         if (mask & s->handler->mask) {
             return s;
         }
@@ -94,7 +150,7 @@
 static void qemu_input_event_trace(QemuConsole *src, InputEvent *evt)
 {
     const char *name;
-    int idx = -1;
+    int qcode, idx = -1;
 
     if (src) {
         idx = qemu_console_get_index(src);
@@ -103,8 +159,10 @@
     case INPUT_EVENT_KIND_KEY:
         switch (evt->key->key->kind) {
         case KEY_VALUE_KIND_NUMBER:
+            qcode = qemu_input_key_number_to_qcode(evt->key->key->number);
+            name = QKeyCode_lookup[qcode];
             trace_input_event_key_number(idx, evt->key->key->number,
-                                         evt->key->down);
+                                         name, evt->key->down);
             break;
         case KEY_VALUE_KIND_QCODE:
             name = QKeyCode_lookup[evt->key->key->qcode];
@@ -133,6 +191,73 @@
     }
 }
 
+static void qemu_input_queue_process(void *opaque)
+{
+    struct QemuInputEventQueueHead *queue = opaque;
+    QemuInputEventQueue *item;
+
+    g_assert(!QTAILQ_EMPTY(queue));
+    item = QTAILQ_FIRST(queue);
+    g_assert(item->type == QEMU_INPUT_QUEUE_DELAY);
+    QTAILQ_REMOVE(queue, item, node);
+    g_free(item);
+
+    while (!QTAILQ_EMPTY(queue)) {
+        item = QTAILQ_FIRST(queue);
+        switch (item->type) {
+        case QEMU_INPUT_QUEUE_DELAY:
+            timer_mod(item->timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL)
+                      + item->delay_ms);
+            return;
+        case QEMU_INPUT_QUEUE_EVENT:
+            qemu_input_event_send(item->src, item->evt);
+            qapi_free_InputEvent(item->evt);
+            break;
+        case QEMU_INPUT_QUEUE_SYNC:
+            qemu_input_event_sync();
+            break;
+        }
+        QTAILQ_REMOVE(queue, item, node);
+        g_free(item);
+    }
+}
+
+static void qemu_input_queue_delay(struct QemuInputEventQueueHead *queue,
+                                   QEMUTimer *timer, uint32_t delay_ms)
+{
+    QemuInputEventQueue *item = g_new0(QemuInputEventQueue, 1);
+    bool start_timer = QTAILQ_EMPTY(queue);
+
+    item->type = QEMU_INPUT_QUEUE_DELAY;
+    item->delay_ms = delay_ms;
+    item->timer = timer;
+    QTAILQ_INSERT_TAIL(queue, item, node);
+
+    if (start_timer) {
+        timer_mod(item->timer, qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL)
+                  + item->delay_ms);
+    }
+}
+
+static void qemu_input_queue_event(struct QemuInputEventQueueHead *queue,
+                                   QemuConsole *src, InputEvent *evt)
+{
+    QemuInputEventQueue *item = g_new0(QemuInputEventQueue, 1);
+
+    item->type = QEMU_INPUT_QUEUE_EVENT;
+    item->src = src;
+    item->evt = evt;
+    QTAILQ_INSERT_TAIL(queue, item, node);
+}
+
+static void qemu_input_queue_sync(struct QemuInputEventQueueHead *queue)
+{
+    QemuInputEventQueue *item = g_new0(QemuInputEventQueue, 1);
+
+    item->type = QEMU_INPUT_QUEUE_SYNC;
+    QTAILQ_INSERT_TAIL(queue, item, node);
+}
+
 void qemu_input_event_send(QemuConsole *src, InputEvent *evt)
 {
     QemuInputHandlerState *s;
@@ -149,7 +274,7 @@
     }
 
     /* send event */
-    s = qemu_input_find_handler(1 << evt->kind);
+    s = qemu_input_find_handler(1 << evt->kind, src);
     if (!s) {
         return;
     }
@@ -192,9 +317,14 @@
 {
     InputEvent *evt;
     evt = qemu_input_event_new_key(key, down);
-    qemu_input_event_send(src, evt);
-    qemu_input_event_sync();
-    qapi_free_InputEvent(evt);
+    if (QTAILQ_EMPTY(&kbd_queue)) {
+        qemu_input_event_send(src, evt);
+        qemu_input_event_sync();
+        qapi_free_InputEvent(evt);
+    } else {
+        qemu_input_queue_event(&kbd_queue, src, evt);
+        qemu_input_queue_sync(&kbd_queue);
+    }
 }
 
 void qemu_input_event_send_key_number(QemuConsole *src, int num, bool down)
@@ -213,6 +343,16 @@
     qemu_input_event_send_key(src, key, down);
 }
 
+void qemu_input_event_send_key_delay(uint32_t delay_ms)
+{
+    if (!kbd_timer) {
+        kbd_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL, qemu_input_queue_process,
+                                 &kbd_queue);
+    }
+    qemu_input_queue_delay(&kbd_queue, kbd_timer,
+                           delay_ms ? delay_ms : kbd_default_delay_ms);
+}
+
 InputEvent *qemu_input_event_new_btn(InputButton btn, bool down)
 {
     InputEvent *evt = g_new0(InputEvent, 1);
@@ -250,7 +390,8 @@
 {
     QemuInputHandlerState *s;
 
-    s = qemu_input_find_handler(INPUT_EVENT_MASK_REL | INPUT_EVENT_MASK_ABS);
+    s = qemu_input_find_handler(INPUT_EVENT_MASK_REL | INPUT_EVENT_MASK_ABS,
+                                NULL);
     return (s != NULL) && (s->handler->mask & INPUT_EVENT_MASK_ABS);
 }
 
diff --git a/ui/sdl2.c b/ui/sdl2.c
index 361de61..fcac87b 100644
--- a/ui/sdl2.c
+++ b/ui/sdl2.c
@@ -49,6 +49,7 @@
     int idx;
     int last_vm_running; /* per console for caption reasons */
     int x, y;
+    int hidden;
 } *sdl2_console;
 
 static SDL_Surface *guest_sprite_surface;
@@ -136,6 +137,9 @@
         } else {
             flags |= SDL_WINDOW_RESIZABLE;
         }
+        if (scon->hidden) {
+            flags |= SDL_WINDOW_HIDDEN;
+        }
 
         scon->real_window = SDL_CreateWindow("", SDL_WINDOWPOS_UNDEFINED,
                                              SDL_WINDOWPOS_UNDEFINED,
@@ -190,30 +194,50 @@
     }
 }
 
-static void reset_keys(void)
+static void reset_keys(struct sdl2_state *scon)
 {
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
     int i;
 
     for (i = 0; i < 256; i++) {
         if (modifiers_state[i]) {
             int qcode = sdl2_scancode_to_qcode[i];
-            qemu_input_event_send_key_qcode(NULL, qcode, false);
+            qemu_input_event_send_key_qcode(con, qcode, false);
             modifiers_state[i] = 0;
         }
     }
 }
 
-static void sdl_process_key(SDL_KeyboardEvent *ev)
+static void sdl_process_key(struct sdl2_state *scon,
+                            SDL_KeyboardEvent *ev)
 {
     int qcode = sdl2_scancode_to_qcode[ev->keysym.scancode];
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
+
+    if (!qemu_console_is_graphic(con)) {
+        if (ev->type == SDL_KEYDOWN) {
+            switch (ev->keysym.scancode) {
+            case SDL_SCANCODE_RETURN:
+                kbd_put_keysym_console(con, '\n');
+                break;
+            case SDL_SCANCODE_BACKSPACE:
+                kbd_put_keysym_console(con, QEMU_KEY_BACKSPACE);
+                break;
+            default:
+                kbd_put_qcode_console(con, qcode);
+                break;
+            }
+        }
+        return;
+    }
 
     switch (ev->keysym.scancode) {
 #if 0
     case SDL_SCANCODE_NUMLOCKCLEAR:
     case SDL_SCANCODE_CAPSLOCK:
         /* SDL does not send the key up event, so we generate it */
-        qemu_input_event_send_key_qcode(NULL, qcode, true);
-        qemu_input_event_send_key_qcode(NULL, qcode, false);
+        qemu_input_event_send_key_qcode(con, qcode, true);
+        qemu_input_event_send_key_qcode(con, qcode, false);
         return;
 #endif
     case SDL_SCANCODE_LCTRL:
@@ -231,7 +255,7 @@
         }
         /* fall though */
     default:
-        qemu_input_event_send_key_qcode(NULL, qcode,
+        qemu_input_event_send_key_qcode(con, qcode,
                                         ev->type == SDL_KEYDOWN);
     }
 }
@@ -302,6 +326,11 @@
 
 static void sdl_grab_start(struct sdl2_state *scon)
 {
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
+
+    if (!con || !qemu_console_is_graphic(con)) {
+        return;
+    }
     /*
      * If the application is not active, do not try to enter grab state. This
      * prevents 'SDL_WM_GrabInput(SDL_GRAB_ON)' from blocking all the
@@ -455,7 +484,7 @@
 
 static void handle_keydown(SDL_Event *ev)
 {
-    int mod_state;
+    int mod_state, win;
     struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
 
     if (alt_grab) {
@@ -470,6 +499,27 @@
 
     if (gui_key_modifier_pressed) {
         switch (ev->key.keysym.scancode) {
+        case SDL_SCANCODE_2:
+        case SDL_SCANCODE_3:
+        case SDL_SCANCODE_4:
+        case SDL_SCANCODE_5:
+        case SDL_SCANCODE_6:
+        case SDL_SCANCODE_7:
+        case SDL_SCANCODE_8:
+        case SDL_SCANCODE_9:
+            win = ev->key.keysym.scancode - SDL_SCANCODE_1;
+            if (win < sdl2_num_outputs) {
+                sdl2_console[win].hidden = !sdl2_console[win].hidden;
+                if (sdl2_console[win].real_window) {
+                    if (sdl2_console[win].hidden) {
+                        SDL_HideWindow(sdl2_console[win].real_window);
+                    } else {
+                        SDL_ShowWindow(sdl2_console[win].real_window);
+                    }
+                }
+                gui_keysym = 1;
+            }
+            break;
         case SDL_SCANCODE_F:
             toggle_full_screen(scon);
             gui_keysym = 1;
@@ -506,7 +556,7 @@
         }
     }
     if (!gui_keysym) {
-        sdl_process_key(&ev->key);
+        sdl_process_key(scon, &ev->key);
     }
 }
 
@@ -531,16 +581,27 @@
             }
             /* SDL does not send back all the modifiers key, so we must
              * correct it. */
-            reset_keys();
+            reset_keys(scon);
             return;
         }
         gui_keysym = 0;
     }
     if (!gui_keysym) {
-        sdl_process_key(&ev->key);
+        sdl_process_key(scon, &ev->key);
     }
 }
 
+static void handle_textinput(SDL_Event *ev)
+{
+    struct sdl2_state *scon = get_scon_from_window(ev->key.windowID);
+    QemuConsole *con = scon ? scon->dcl.con : NULL;
+
+    if (qemu_console_is_graphic(con)) {
+        return;
+    }
+    kbd_put_string_console(con, ev->text.text, strlen(ev->text.text));
+}
+
 static void handle_mousemotion(SDL_Event *ev)
 {
     int max_x, max_y;
@@ -677,6 +738,9 @@
         case SDL_KEYUP:
             handle_keyup(ev);
             break;
+        case SDL_TEXTINPUT:
+            handle_textinput(ev);
+            break;
         case SDL_QUIT:
             if (!no_quit) {
                 no_shutdown = 0;
@@ -805,7 +869,7 @@
 
     for (i = 0;; i++) {
         QemuConsole *con = qemu_console_lookup_by_index(i);
-        if (!con || !qemu_console_is_graphic(con)) {
+        if (!con) {
             break;
         }
     }
@@ -813,6 +877,9 @@
     sdl2_console = g_new0(struct sdl2_state, sdl2_num_outputs);
     for (i = 0; i < sdl2_num_outputs; i++) {
         QemuConsole *con = qemu_console_lookup_by_index(i);
+        if (!qemu_console_is_graphic(con)) {
+            sdl2_console[i].hidden = true;
+        }
         sdl2_console[i].dcl.ops = &dcl_ops;
         sdl2_console[i].dcl.con = con;
         register_displaychangelistener(&sdl2_console[i].dcl);
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 59b59c0..f02352c 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -181,6 +181,10 @@
         }
     }
 
+    if (pixels == 0) {
+        return 0;
+    }
+
     /* 95% smooth or more ... */
     if (stats[0] * 33 / pixels >= 95) {
         return 0;
@@ -267,7 +271,9 @@
                 y += w;                                                 \
             }                                                           \
         }                                                               \
-                                                                        \
+        if (pixels == 0) {                                              \
+            return 0;                                                   \
+        }                                                               \
         if ((stats[0] + stats[1]) * 100 / pixels >= 90) {               \
             return 0;                                                   \
         }                                                               \
diff --git a/ui/vnc.c b/ui/vnc.c
index 2d7def9..1684206 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -26,6 +26,7 @@
 
 #include "vnc.h"
 #include "vnc-jobs.h"
+#include "trace.h"
 #include "sysemu/sysemu.h"
 #include "qemu/sockets.h"
 #include "qemu/timer.h"
@@ -1552,7 +1553,9 @@
 {
     int keycode = keysym2scancode(vs->vd->kbd_layout, keysym) & SCANCODE_KEYMASK;
     qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, true);
+    qemu_input_event_send_key_delay(0);
     qemu_input_event_send_key_number(vs->vd->dcl.con, keycode, false);
+    qemu_input_event_send_key_delay(0);
 }
 
 static int current_led_state(VncState *vs)
@@ -1597,6 +1600,10 @@
     int caps, num, scr;
     bool has_changed = (ledstate != current_led_state(vs));
 
+    trace_vnc_key_guest_leds((ledstate & QEMU_CAPS_LOCK_LED),
+                             (ledstate & QEMU_NUM_LOCK_LED),
+                             (ledstate & QEMU_SCROLL_LOCK_LED));
+
     caps = ledstate & QEMU_CAPS_LOCK_LED ? 1 : 0;
     num  = ledstate & QEMU_NUM_LOCK_LED  ? 1 : 0;
     scr  = ledstate & QEMU_SCROLL_LOCK_LED ? 1 : 0;
@@ -1659,11 +1666,13 @@
         */
         if (keysym_is_numlock(vs->vd->kbd_layout, sym & 0xFFFF)) {
             if (!vs->modifiers_state[0x45]) {
+                trace_vnc_key_sync_numlock(true);
                 vs->modifiers_state[0x45] = 1;
                 press_key(vs, 0xff7f);
             }
         } else {
             if (vs->modifiers_state[0x45]) {
+                trace_vnc_key_sync_numlock(false);
                 vs->modifiers_state[0x45] = 0;
                 press_key(vs, 0xff7f);
             }
@@ -1682,11 +1691,13 @@
         int capslock = !!(vs->modifiers_state[0x3a]);
         if (capslock) {
             if (uppercase == shift) {
+                trace_vnc_key_sync_capslock(false);
                 vs->modifiers_state[0x3a] = 0;
                 press_key(vs, 0xffe5);
             }
         } else {
             if (uppercase != shift) {
+                trace_vnc_key_sync_capslock(true);
                 vs->modifiers_state[0x3a] = 1;
                 press_key(vs, 0xffe5);
             }
@@ -1819,6 +1830,11 @@
     }
 }
 
+static const char *code2name(int keycode)
+{
+    return QKeyCode_lookup[qemu_input_key_number_to_qcode(keycode)];
+}
+
 static void key_event(VncState *vs, int down, uint32_t sym)
 {
     int keycode;
@@ -1829,6 +1845,7 @@
     }
 
     keycode = keysym2scancode(vs->vd->kbd_layout, lsym & 0xFFFF) & SCANCODE_KEYMASK;
+    trace_vnc_key_event_map(down, sym, keycode, code2name(keycode));
     do_key_event(vs, down, keycode, sym);
 }
 
@@ -1836,10 +1853,12 @@
                           uint32_t sym, uint16_t keycode)
 {
     /* if the user specifies a keyboard layout, always use it */
-    if (keyboard_layout)
+    if (keyboard_layout) {
         key_event(vs, down, sym);
-    else
+    } else {
+        trace_vnc_key_event_ext(down, sym, keycode, code2name(keycode));
         do_key_event(vs, down, keycode, sym);
+    }
 }
 
 static void framebuffer_update_request(VncState *vs, int incremental,
@@ -2929,10 +2948,12 @@
     QTAILQ_INIT(&vs->clients);
     vs->expires = TIME_MAX;
 
-    if (keyboard_layout)
+    if (keyboard_layout) {
+        trace_vnc_key_map_init(keyboard_layout);
         vs->kbd_layout = init_keyboard_layout(name2keysym, keyboard_layout);
-    else
+    } else {
         vs->kbd_layout = init_keyboard_layout(name2keysym, "en-us");
+    }
 
     if (!vs->kbd_layout)
         exit(1);
@@ -2976,26 +2997,6 @@
 #endif
 }
 
-static int vnc_display_disable_login(DisplayState *ds)
-{
-    VncDisplay *vs = vnc_display;
-
-    if (!vs) {
-        return -1;
-    }
-
-    if (vs->password) {
-        g_free(vs->password);
-    }
-
-    vs->password = NULL;
-    if (vs->auth == VNC_AUTH_NONE) {
-        vs->auth = VNC_AUTH_VNC;
-    }
-
-    return 0;
-}
-
 int vnc_display_password(DisplayState *ds, const char *password)
 {
     VncDisplay *vs = vnc_display;
@@ -3003,20 +3004,18 @@
     if (!vs) {
         return -EINVAL;
     }
-
-    if (!password) {
-        /* This is not the intention of this interface but err on the side
-           of being safe */
-        return vnc_display_disable_login(ds);
+    if (vs->auth == VNC_AUTH_NONE) {
+        error_printf_unless_qmp("If you want use passwords please enable "
+                                "password auth using '-vnc ${dpy},password'.");
+        return -EINVAL;
     }
 
     if (vs->password) {
         g_free(vs->password);
         vs->password = NULL;
     }
-    vs->password = g_strdup(password);
-    if (vs->auth == VNC_AUTH_NONE) {
-        vs->auth = VNC_AUTH_VNC;
+    if (password) {
+        vs->password = g_strdup(password);
     }
 
     return 0;
diff --git a/user-exec.c b/user-exec.c
index 8ed6fec..1ff8673 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -21,6 +21,7 @@
 #include "disas/disas.h"
 #include "tcg.h"
 #include "qemu/bitops.h"
+#include "exec/cpu_ldst.h"
 
 #undef EAX
 #undef ECX
diff --git a/util/error.c b/util/error.c
index 66245cc..2ace0d8 100644
--- a/util/error.c
+++ b/util/error.c
@@ -142,11 +142,6 @@
     return err_new;
 }
 
-bool error_is_set(Error **errp)
-{
-    return (errp && *errp);
-}
-
 ErrorClass error_get_class(const Error *err)
 {
     return err->err_class;
diff --git a/util/iov.c b/util/iov.c
index 6569b5a..49f8838 100644
--- a/util/iov.c
+++ b/util/iov.c
@@ -335,6 +335,27 @@
     qemu_iovec_concat_iov(dst, src->iov, src->niov, soffset, sbytes);
 }
 
+/*
+ * Check if the contents of the iovecs are all zero
+ */
+bool qemu_iovec_is_zero(QEMUIOVector *qiov)
+{
+    int i;
+    for (i = 0; i < qiov->niov; i++) {
+        size_t offs = QEMU_ALIGN_DOWN(qiov->iov[i].iov_len, 4 * sizeof(long));
+        uint8_t *ptr = qiov->iov[i].iov_base;
+        if (offs && !buffer_is_zero(qiov->iov[i].iov_base, offs)) {
+            return false;
+        }
+        for (; offs < qiov->iov[i].iov_len; offs++) {
+            if (ptr[offs]) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
 void qemu_iovec_destroy(QEMUIOVector *qiov)
 {
     assert(qiov->nalloc != -1);
diff --git a/util/qemu-sockets.c b/util/qemu-sockets.c
index 8818d7c..627e609 100644
--- a/util/qemu-sockets.c
+++ b/util/qemu-sockets.c
@@ -354,6 +354,7 @@
 int inet_connect_opts(QemuOpts *opts, Error **errp,
                       NonBlockingConnectHandler *callback, void *opaque)
 {
+    Error *local_err = NULL;
     struct addrinfo *res, *e;
     int sock = -1;
     bool in_progress;
@@ -372,24 +373,27 @@
     }
 
     for (e = res; e != NULL; e = e->ai_next) {
-        if (error_is_set(errp)) {
-            error_free(*errp);
-            *errp = NULL;
-        }
+        error_free(local_err);
+        local_err = NULL;
         if (connect_state != NULL) {
             connect_state->current_addr = e;
         }
-        sock = inet_connect_addr(e, &in_progress, connect_state, errp);
-        if (in_progress) {
-            return sock;
-        } else if (sock >= 0) {
-            /* non blocking socket immediate success, call callback */
-            if (callback != NULL) {
-                callback(sock, opaque);
-            }
+        sock = inet_connect_addr(e, &in_progress, connect_state, &local_err);
+        if (sock >= 0) {
             break;
         }
     }
+
+    if (sock < 0) {
+        error_propagate(errp, local_err);
+    } else if (in_progress) {
+        /* wait_for_connect() will do the rest */
+        return sock;
+    } else {
+        if (callback) {
+            callback(sock, opaque);
+        }
+    }
     g_free(connect_state);
     freeaddrinfo(res);
     return sock;
diff --git a/vl.c b/vl.c
index bda88f7..a3def97 100644
--- a/vl.c
+++ b/vl.c
@@ -965,7 +965,7 @@
     return 0;
 }
 
-static void parse_name(QemuOpts *opts)
+static int parse_name(QemuOpts *opts, void *opaque)
 {
     const char *proc_name;
 
@@ -978,6 +978,8 @@
     if (proc_name) {
         os_set_proc_name(proc_name);
     }
+
+    return 0;
 }
 
 bool usb_enabled(bool default_usb)
@@ -2889,7 +2891,8 @@
     StringInputVisitor *siv;
     Error *local_err = NULL;
 
-    if (strcmp(name, "qom-type") == 0 || strcmp(name, "id") == 0) {
+    if (strcmp(name, "qom-type") == 0 || strcmp(name, "id") == 0 ||
+        strcmp(name, "type") == 0) {
         return 0;
     }
 
@@ -3796,7 +3799,6 @@
                 if (!opts) {
                     exit(1);
                 }
-                parse_name(opts);
                 break;
             case QEMU_OPTION_prom_env:
                 if (nb_prom_envs >= MAX_PROM_ENVS) {
@@ -3971,6 +3973,10 @@
         exit(1);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("name"), parse_name, NULL, 1)) {
+        exit(1);
+    }
+
 #ifndef _WIN32
     if (qemu_opts_foreach(qemu_find_opts("add-fd"), parse_add_fd, NULL, 1)) {
         exit(1);
@@ -4214,6 +4220,13 @@
         exit(0);
     }
 
+    machine_opts = qemu_get_machine_opts();
+    if (qemu_opt_foreach(machine_opts, object_set_property, current_machine,
+                         1) < 0) {
+        object_unref(OBJECT(current_machine));
+        exit(1);
+    }
+
     configure_accelerator(machine_class);
 
     if (qtest_chrdev) {
@@ -4258,6 +4271,7 @@
 
     if (!kernel_cmdline) {
         kernel_cmdline = "";
+        current_machine->kernel_cmdline = (char *)kernel_cmdline;
     }
 
     linux_boot = (kernel_filename != NULL);
@@ -4420,16 +4434,11 @@
 
     qdev_machine_init();
 
-    current_machine->init_args = (QEMUMachineInitArgs) {
-        .machine = machine_class,
-        .ram_size = ram_size,
-        .boot_order = boot_order,
-        .kernel_filename = kernel_filename,
-        .kernel_cmdline = kernel_cmdline,
-        .initrd_filename = initrd_filename,
-        .cpu_model = cpu_model };
+    current_machine->ram_size = ram_size;
+    current_machine->boot_order = boot_order;
+    current_machine->cpu_model = cpu_model;
 
-    machine_class->init(&current_machine->init_args);
+    machine_class->init(current_machine);
 
     audio_init();